diff --git a/package-mac/ebi-package.command b/package-mac/ebi-package.command old mode 100644 new mode 100755 index e03ccbb..878dc08 --- a/package-mac/ebi-package.command +++ b/package-mac/ebi-package.command @@ -11,9 +11,8 @@ cp source/ebi/gtk-help.png dist/ebi.app/Contents/Resources/lib/. cp source/ebi/gtk-stop.png dist/ebi.app/Contents/Resources/lib/. cp source/ebi/options.png dist/ebi.app/Contents/Resources/lib/. cp source/ebi/pause.png dist/ebi.app/Content -frjikjhjkfdf021s/Resources/lib/. cp ebi.command dist/ebi.app/Contents/MacOS/. cd dist hdiutil create -srcfolder ebi.app ebi.dmg -hdiutil internet-enable -yes ebi.dmg \ No newline at end of file +hdiutil internet-enable -yes ebi.dmg diff --git a/package-mac/ebi.command b/package-mac/ebi.command old mode 100644 new mode 100755 diff --git a/package-mac/source/ebi/Engine.py b/package-mac/source/ebi/Engine.py new file mode 100644 index 0000000..1f738e9 --- /dev/null +++ b/package-mac/source/ebi/Engine.py @@ -0,0 +1,3508 @@ +# Engine.py +# +# Author: Jim Kurian, Pearson plc. +# Date: October 2014 +# +# The CSV processing engine of the EQUELLA Bulk Importer. Loads a +# CSV file and iterates through the rows creating items in EQUELLA +# for each one. Utilizes equellaclient41.py for EQUELLA +# communications. Invoked by Mainframe.py. + +from xml.dom import Node +from equellaclient41 import * +import time, datetime +import zipfile, csv, codecs, cStringIO +import sys, platform +import traceback +import random +import os +import zipfile, glob, time, getpass, uuid +import wx + +class Engine(): + def __init__(self, owner, Version, Copyright): + # constants + self.METADATA = 'Metadata' + self.ATTACHMENTLOCATIONS = 'Attachment Locations' + self.ATTACHMENTNAMES = 'Attachment Names' + self.CUSTOMATTACHMENTS = 'Custom Attachments' + self.RAWFILES = 'Raw Files' + self.URLS = 'URLs' + self.HYPERLINKNAMES = 'Hyperlink Names' + self.EQUELLARESOURCES = 'EQUELLA Resources' + self.EQUELLARESOURCENAMES = 'EQUELLA Resource Names' + self.COMMANDS = 'Commands' + self.TARGETIDENTIFIER = 'Target Identifier' + self.TARGETVERSION = 'Target Version' + self.COLLECTION = 'Collection' + self.OWNER = 'Owner' + self.COLLABORATORS = 'Collaborators' + self.ITEMID = 'Item ID' + self.ITEMVERSION = 'Item Version' + self.ROWERROR = 'Row Error' + self.THUMBNAILS = "Thumbnails" + self.SELECTEDTHUMBNAIL = "Selected Thumbnail" + self.IGNORE = 'Ignore' + + self.COLUMN_POS = "Pos" + self.COLUMN_HEADING = "Column Heading" + self.COLUMN_DATATYPE = "Column Data Type" + self.COLUMN_DISPLAY = "Display" + self.COLUMN_SOURCEIDENTIFIER = "Source Identifier" + self.COLUMN_XMLFRAGMENT = "XML Fragment" + self.COLUMN_DELIMITER = "Delimiter" + + self.CLEARMETA = 0 + self.REPLACEMETA = 1 + self.APPENDMETA = 2 + + self.OVERWRITENONE = 0 + self.OVERWRITEEXISTING = 1 + self.OVERWRITEALL = 2 + + self.pause = False + self.owner = owner + + # default settings (can be overridden in ebi.properties) + self.debug = False + self.attachmentMetadataTargets = True + self.defaultChunkSize = (1024 * 2048) + self.chunkSize = self.defaultChunkSize + self.networkLogging = False + self.scormformatsupport = True + + self.copyright = Copyright + self.rowFilter = "" + self.logfilesfolder = "logs" + self.logfilespath = "" + self.testItemfolder = "test_output" + self.receiptFolder = "receipts" + self.sessionName = "" + self.maxRetry = 5 + + # welcome message for command prompt and log files + self.welcomeLine1 = "EQUELLA Bulk Importer [EBI %s, %s]" % (Version, self.getPlatform()) + self.welcomeLine2 = self.copyright + "\n" + + print self.welcomeLine1 + print self.welcomeLine2 + + # CSV and connection settings + self.institutionUrl = "" + self.username = "" + self.password = "" + self.collection = "" + self.csvFilePath = "" + + # Options + self.proxy = "" + self.proxyUsername = "" + self.proxyPassword = "" + self.encoding = "utf8" + self.saveTestXML = False + self.saveAsDraft = False + self.saveTestXml = False + self.existingMetadataMode = self.CLEARMETA + self.appendAttachments = False + self.createNewVersions = False + self.useEBIUsername = False + self.ignoreNonexistentCollaborators = False + self.saveNonexistentUsernamesAsIDs = True + self.attachmentsBasepath = "" + self.absoluteAttachmentsBasepath = "" + self.export = False + self.includeNonLive = False + self.overwriteMode = self.OVERWRITENONE + self.whereClause = "" + self.startScript = "" + self.endScript = "" + self.preScript = "" + self.postScript = "" + + self.ebiScriptObject = EbiScriptObject(self) + + # data structures to store column settings + self.currentColumns = [] + self.csvArray = [] + + self.successCount = 0 + self.errorCount = 0 + + + # enum for attachment types + self.attachmentTypeFile = 0 + self.attachmentTypeZip = 1 + self.attachmentTypeIMS = 2 + self.attachmentTypeSCORM = 3 + + self.columnHeadings = [] + self.StopProcessing = False + self.processingStoppedByScript = False + self.Skip = False + self.logFileName = "" + self.collectionIDs = {} + + self.itemSystemNodes = [ + "staging", + "name", + "description", + "itemdefid", + "datecreated", + "datemodified", + "dateforindex", + "owner", + "collaborativeowners", + "rating", + "badurls", + "moderation", + "newitem", + "attachments", + "navigationnodes", + "url", + "history", + "thumbnail" + ] + self.sourceIdentifierReceipts = {} + self.exportedFiles = [] + self.eqVersionmm = "" + self.eqVersionmmr = "" + self.eqVersionDisplay = "" + + def getPlatform(self): + ebiPlatform = "Python " + platform.python_version() + system = platform.system() + if system == "Windows": + ebiPlatform += ", Windows " + platform.release() + elif system == "Darwin": + ebiPlatform += ", Mac OS " + platform.mac_ver()[0] + elif system == "Linux": + ebiPlatform += ", " + platform.linux_distribution()[0] + " " + platform.linux_distribution()[1] + return ebiPlatform + + def setDebug(self, debug): + self.debug = debug + if self.debug: + self.echo("debug = True") + + def setLog(self, log): + self.log = log + self.log.AddLogText(self.welcomeLine1 + "\n", 1) + self.log.AddLogText(self.welcomeLine2 + "\n", 1) + + def echo(self, entry, display = True, log = True, style = 0): + + if log and self.logFileName != "": + + # create/open log file + logfile = open(os.path.join(self.logfilespath, self.logFileName),"a") + + # write entry + logfile.writelines(entry.encode(self.encoding) + "\n") + + logfile.close() + + if display: + self.log.AddLogText(entry.encode(self.encoding) + "\n", style) + print entry.encode(self.encoding) + return + + def translateError(self, rawError, context = ""): + + rawError = str(rawError) + + # check if it is a SOAP error + if rawError.rfind('') != -1: + # Extract faultstring from 500 code and display/log + rawError = rawError[rawError.find('faultstring') + 12:rawError.rfind('") + # iterate through columns and check for validity + for n, columnHeading in enumerate(self.columnHeadings): + if self.currentColumns[n][self.COLUMN_DATATYPE] == self.METADATA: + if columnHeading == "": + raise Exception, "Blank column heading found on column requiring XPath '%s' (column %s)" % (columnHeading, n + 1) + + if self.currentColumns[n][self.COLUMN_DATATYPE] == self.METADATA or \ + (self.currentColumns[n][self.COLUMN_DATATYPE] in [self.ATTACHMENTLOCATIONS, self.URLS, self.EQUELLARESOURCES, self.CUSTOMATTACHMENTS] and \ + columnHeading.strip() != "" and columnHeading.strip()[0] != "#"): + try: + # test xpath + testPB.validateXpath(columnHeading.strip()) + except: + if self.debug: + raise + else: + exceptionValue = sys.exc_info()[1] + scriptErrorMsg = "Invalid column heading '%s' (column %s). %s" % (columnHeading, n + 1, exceptionValue) + raise Exception, scriptErrorMsg + + # warn the user if any XPaths are attempting to overwrite system nodes + xpathParts = columnHeading.split("/") + if columnHeading.strip() == "item" or len(xpathParts) > 1: + if columnHeading.strip() == "item" or (xpathParts[0].strip() == "item" and xpathParts[1].strip() in self.itemSystemNodes): + self.echo("WARNING: XPath '%s' in column %s is writing to a system node" % (columnHeading, n + 1)) + + def getEquellaVersion(self): + # download and read version.properties + try: + versionUrl = self.institutionUrl + "/version.properties" + versionProperties = "" + versionProperties = self.tle.getText(versionUrl) + vpLines = versionProperties.split("\n") + + for line in vpLines: + line = line.strip() + lineparts = line.split("=") + if lineparts[0] == "version.mm": + self.eqVersionmm = lineparts[1] + if lineparts[0] == "version.mmr": + self.eqVersionmmr = lineparts[1] + if lineparts[0] == "version.display": + self.eqVersionDisplay = lineparts[1] + + except: + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + self.echo(''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback))) + + def getContributableCollections(self): + try: + # connect to EQUELLA + self.tle = TLEClient(self, self.institutionUrl, self.username, self.password, self.proxy, self.proxyUsername, self.proxyPassword, self.debug) + self.getEquellaVersion() + except: + if self.debug: + raise + else: + raise Exception, self.translateError(str(sys.exc_info()[1]), "login") + + try: + # get all contributable collections and their IDs + itemDefs = self.tle._enumerateItemDefs() + self.collectionIDs.clear() + for key, value in itemDefs.items(): + self.collectionIDs[key] = value["uuid"] + + self.tle.logout() + + # return collection names (sorted) + return sorted(self.collectionIDs.keys()) + except: + if self.debug: + raise + else: + raise Exception, self.translateError(str(sys.exc_info()[1])) + + # find index of first matching instance of value, return -1 if not found + def lookupColumnIndex(self, columnProperty, value): + for index, column in enumerate(self.currentColumns): + if column[columnProperty] == value: + + # value found + return index + + # value not found + return -1 + + def tryPausing(self, message, newline = False): + progress = message + if self.pause: + self.log.Enable() + # add message to log + self.log.SetReadOnly(False) + if newline: + self.log.AppendText("\n") + self.log.AppendText(progress) + self.log.SetReadOnly(True) + self.log.GotoPos(self.log.GetLength()) + statusOriginalText = self.owner.mainStatusBar.GetStatusText(0) + self.owner.mainStatusBar.SetStatusText("PAUSED...", 2) + + # pause loop + count = 0 + while self.pause: + wx.GetApp().Yield() + time.sleep(0.5) + if count == 0: + self.owner.mainStatusBar.SetStatusText("PAUSED.", 2) + count = 1 + elif count == 1: + self.owner.mainStatusBar.SetStatusText("PAUSED..", 2) + count = 2 + else: + self.owner.mainStatusBar.SetStatusText("PAUSED...", 2) + count = 0 + + + # remove message + if message != "": + self.log.DocumentEnd() + self.log.SetReadOnly(False) + for i in range(len(progress)): + self.log.DeleteBack() + if newline: + self.log.DeleteBack() + self.log.SetReadOnly(True) + self.log.Disable() + self.owner.mainStatusBar.SetStatusText("", 2) + + def runImport(self, owner, testOnly=False): + self.StopProcessing = False + self.pause = False + self.processingStoppedByScript = False + + try: + # try opening csv file + if owner.txtCSVPath.GetValue() != "" and not os.path.isdir(self.csvFilePath): + f = open(self.csvFilePath, "rb") + f.close() + except: + owner.mainStatusBar.SetStatusText("Processing halted due to an error", 0) + raise Exception, "CSV file could not be opened, check path: %s" % self.csvFilePath + + # specify folders + self.logfilespath = os.path.join(os.path.dirname(self.csvFilePath), self.logfilesfolder) + self.testItemfolder = os.path.join(os.path.dirname(self.csvFilePath), self.testItemfolder) + self.receiptFolder = os.path.join(os.path.dirname(self.csvFilePath), self.receiptFolder) + + # specify log file name for this run + if not os.path.exists(self.logfilespath): + os.makedirs(self.logfilespath) + self.sessionName = datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + self.logFileName = self.sessionName + '.txt' + + self.echo(self.welcomeLine1, False) + self.echo(self.welcomeLine2, False) + if self.debug: + self.echo("Debug mode on\n", False) + + # create objects for EBI scripts + self.logger = Logger(self) + self.process = Process(self) + + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "Opening a connection to EQUELLA at %s..." % self.institutionUrl) + + try: + + # set stats counters + self.successCount = 0 + self.errorCount = 0 + + # connect to EQUELLA + owner.mainStatusBar.SetStatusText("Connecting...", 0) + wx.GetApp().Yield() + self.tle = TLEClient(self, self.institutionUrl, self.username, self.password, self.proxy, self.proxyUsername, self.proxyPassword, self.debug) + + # get EQUELLA version + if self.eqVersionmm == "": + self.getEquellaVersion() + versionDisplay = "" + if self.eqVersionDisplay != "": + versionDisplay = " (%s)" % self.eqVersionDisplay + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "Successfully connected to EQUELLA%s" % versionDisplay) + + # Get Collection UUID + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "Target collection: '" + self.collection + "'...") + + # if not previously retreieved get all contributable collections and their IDs + if len(self.collectionIDs) == 0: + itemDefs = self.tle._enumerateItemDefs() + for key, value in itemDefs.items(): + self.collectionIDs[key] = value["uuid"] + + # get ID of selected collection + if self.collection in self.collectionIDs.keys(): + itemdefuuid = self.collectionIDs[self.collection] + else: + raise Exception, "Collection '" + self.collection + "'" + " not found" + + try: + if not os.path.isdir(self.csvFilePath): + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "Parsing CSV file (" + self.csvFilePath + ")...") + else: + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "WARNING: No CSV specified. CSV path is " + self.csvFilePath) + + if not owner.verifyCurrentColumnsMatchCSV(): + raise Exception, 'CSV headings do not match the settings, update the settings to match the CSV column headings' + + # determine the column indexes for the following column types + sourceIdentifierColumn = self.lookupColumnIndex(self.COLUMN_SOURCEIDENTIFIER, "YES") + targetIdentifierColumn = self.lookupColumnIndex(self.COLUMN_DATATYPE, self.TARGETIDENTIFIER) + targetVersionColumn = self.lookupColumnIndex(self.COLUMN_DATATYPE, self.TARGETVERSION) + commandOptionsColumn = self.lookupColumnIndex(self.COLUMN_DATATYPE, self.COMMANDS) + attachmentLocationsColumn = self.lookupColumnIndex(self.COLUMN_DATATYPE, self.ATTACHMENTLOCATIONS) + customAttachmentsColumn = self.lookupColumnIndex(self.COLUMN_DATATYPE, self.CUSTOMATTACHMENTS) + urlsColumn = self.lookupColumnIndex(self.COLUMN_DATATYPE, self.URLS) + resourcesColumn = self.lookupColumnIndex(self.COLUMN_DATATYPE, self.EQUELLARESOURCES) + itemIdColumn = self.lookupColumnIndex(self.COLUMN_DATATYPE, self.ITEMID) + versionColumn = self.lookupColumnIndex(self.COLUMN_DATATYPE, self.ITEMVERSION) + thumbnailsColumn = self.lookupColumnIndex(self.COLUMN_DATATYPE, self.THUMBNAILS) + selectedThumbnailColumn = self.lookupColumnIndex(self.COLUMN_DATATYPE, self.SELECTEDTHUMBNAIL) + rowErrorColumn = self.lookupColumnIndex(self.COLUMN_DATATYPE, self.ROWERROR) + collectionColumn = self.lookupColumnIndex(self.COLUMN_DATATYPE, self.COLLECTION) + ownerColumn = self.lookupColumnIndex(self.COLUMN_DATATYPE, self.OWNER) + collaboratorsColumn = self.lookupColumnIndex(self.COLUMN_DATATYPE, self.COLLABORATORS) + + # ignore Source Identifier if column datatype is set to Ignore + if sourceIdentifierColumn != -1 and self.currentColumns[sourceIdentifierColumn][self.COLUMN_DATATYPE] == self.IGNORE: + sourceIdentifierColumn = -1 + + # parse CSV + self.csvParse(owner, + self.tle, + itemdefuuid, + testOnly, + sourceIdentifierColumn, + targetIdentifierColumn, + targetVersionColumn, + commandOptionsColumn, + attachmentLocationsColumn, + urlsColumn, + resourcesColumn, + customAttachmentsColumn, + itemIdColumn, + versionColumn, + rowErrorColumn, + collectionColumn, + thumbnailsColumn, + selectedThumbnailColumn, + ownerColumn, + collaboratorsColumn) + + except: + owner.mainStatusBar.SetStatusText("Processing halted due to an error", 0) + + err = str(sys.exc_info()[1]) + exact_error = err + + errorString = "" + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + if self.debug: + errorString = "\n" + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + + # check if it is a SOAP error + if err.rfind('') != -1: + # Extract faultstring from 500 code and display/log + exact_error = err[err.find('faultstring') + 12:err.rfind('') != -1: + # Extract faultstring from 500 code and display/log + exact_error = err[err.find('faultstring') + 12:err.rfind(' 0: + break + else: + del self.csvArray[-1] + + # if CSV file is empty or non-existent populate the first row with column headings from settings + if len(self.csvArray) == 0: + self.csvArray.append([]) + for columnHeading in self.columnHeadings: + self.csvArray[0].append(columnHeading) + + def csvParse(self, + owner, + tle, + itemdefuuid, + testOnly, + sourceIdentifierColumn, + targetIdentifierColumn, + targetVersionColumn, + commandOptionsColumn, + attachmentLocationsColumn, + urlsColumn, + resourcesColumn, + customAttachmentsColumn, + itemIdColumn, + versionColumn, + rowErrorColumn, + collectionColumn, + thumbnailsColumn, + selectedThumbnailColumn, + ownerColumn, + collaboratorsColumn): + + # if real form receipt filename and run check if receipts file is editable + receiptFilename = "" + if not self.export: + if itemIdColumn != -1: + # form receipts filename + if owner.txtCSVPath.GetValue() != "" and not os.path.isdir(self.csvFilePath): + receiptFilename = os.path.join(self.receiptFolder, os.path.basename(self.csvFilePath)) + else: + receiptFilename = os.path.join(self.receiptFolder, "receipt.csv") + + if os.path.exists(receiptFilename): + try: + # try opening file for editing + f = open(receiptFilename, "wb") + f.close() + except: + raise Exception, "Receipts file cannot be written to and may be in use: %s" % receiptFilename + + + # read the CSV and store the rows in an array + self.loadCSV(owner) + + # warn if not using attachment metadata targets + if not self.attachmentMetadataTargets: + self.echo("\nWARNING: Not using attachments metadata targets (not suitable for EQUELLA 5.0 or higher)\n") + + # calculate absolute appachments basepath for attachments + self.absoluteAttachmentsBasepath = os.path.join(os.path.dirname(self.csvFilePath), self.attachmentsBasepath.strip()) + if self.debug: + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "Absolute attachments basepath is " + self.absoluteAttachmentsBasepath) + + # indicate what scripts, if any, are present + scriptsPresent = [] + if self.startScript.strip() != "": + scriptsPresent.append("Start Script") + if self.preScript.strip() != "": + scriptsPresent.append("Row Pre-Script") + if self.postScript.strip() != "": + scriptsPresent.append("Row Post-Script") + if self.endScript.strip() != "": + scriptsPresent.append("End Script") + if len(scriptsPresent) > 0: + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "Scripts present: " + ", ".join(scriptsPresent)) + + # set variables for scripts + self.scriptVariables = {} + if self.export: + action = 1 + else: + action = 0 + + # run Start Script + if self.startScript.strip() != "" and not self.export: + try: + exec self.startScript in { + "IMPORT":0, + "EXPORT":1, + "mode":action, + "vars":self.scriptVariables, + "testOnly": testOnly, + "institutionUrl":tle.institutionUrl, + "collection":self.collection, + "csvFilePath":self.csvFilePath, + "username":self.username, + "logger":self.logger, + "columnHeadings":self.columnHeadings, + "columnSettings":self.currentColumns, + "successCount":self.successCount, + "errorCount":self.errorCount, + "process":self.process, + "basepath":self.absoluteAttachmentsBasepath, + "sourceIdentifierIndex":sourceIdentifierColumn, + "targetIdentifierIndex":targetIdentifierColumn, + "targetVersionIndex":targetVersionColumn, + "csvData":self.csvArray, + "ebi":self.ebiScriptObject, + "equella":tle, + } + except: + if self.debug: + raise + else: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + formattedException = "".join(traceback.format_exception_only(exceptionType, exceptionValue))[:-1] + scriptErrorMsg = "An error occured in the Start Script:\n%s (line %s)" % (formattedException, traceback.extract_tb(exceptionTraceback)[-1][1]) + raise Exception, scriptErrorMsg + + if not self.export: + self.validateColumnHeadings() + + # set all rows to be processed + scheduledRows = range(1, len(self.csvArray)) + rowsToBeProcessedCount = len(self.csvArray) - 1 + scheduledRowsLabel = "all rows to be processed" + + # check if row filter applies + if self.rowFilter.strip() != "": + try: + scheduledRows = [] + + # populate scheduledRows based on rowsFilter + rowRanges = self.rowFilter.split(",") + for rowRange in rowRanges: + rows = rowRange.split("-") + + if len(rows) == 1: + + # single row number encountered + scheduledRows.append(int(rows[0])) + + if len(rows) == 2: + + # row range provided + if rows[1].strip() == "": + + # no finish row so assume all remaining rows (e.g. "5-") + rows[1] = len(self.csvArray) - 1 + + scheduledRows.extend(range(int(rows[0]), int(rows[1]) + 1)) + + # remove any duplicates (preserving order) + scheduledRows = self.removeDuplicates(scheduledRows) + + # determine how many rows to be processed + rowsToBeProcessedCount = 0 + for rc in scheduledRows: + if rc < len(self.csvArray): + rowsToBeProcessedCount += 1 + + # form label for how many rows to be processed + scheduledRowsLabel = "%s to be processed [%s]" % (rowsToBeProcessedCount, self.rowFilter) + + except: + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + self.echo(''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback))) + raise Exception, "Invalid row filter specified" + + if not self.export: + # echo rows to be processed + if testOnly: + actionString = "%s row(s) found, %s (test only)" % (len(self.csvArray) - 1 if len(self.csvArray) > 0 else 0, scheduledRowsLabel) + else: + actionString = "%s row(s) found, %s" % (len(self.csvArray) - 1 if len(self.csvArray) > 0 else 0, scheduledRowsLabel) + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + actionString) + + # echo draft and new version settings + actionString = "" + if sourceIdentifierColumn != -1 or targetIdentifierColumn != -1: + if self.saveAsDraft and self.createNewVersions: + actionString = "Options -> Create new versions of existing items in draft status" + elif self.createNewVersions: + actionString = "Options -> Create new versions of existing items" + elif self.saveAsDraft: + actionString = "Options -> Create new items in draft status (status of existing items will remain unchanged)" + elif self.saveAsDraft: + actionString = "Options -> Create items in draft status" + if actionString != "": + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + actionString) + + # echo append/replace metadata settings + actionString = "" + if sourceIdentifierColumn != -1 or targetIdentifierColumn != -1: + if self.existingMetadataMode == self.APPENDMETA: + actionString = "Options -> Append metadata to existing items" + if self.existingMetadataMode == self.REPLACEMETA: + actionString = "Options -> Replace specified metadata in existing items" + if actionString != "": + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + actionString) + + # echo append attachment settings + if self.appendAttachments: + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "Options -> Append attachments to existing items") + + + # iterate through the rows of metadata from the CSV file creating an item in EQUELLA for each + rowReceipts = {} + processedCounter = 0 + self.sourceIdentifierReceipts = {} + + self.owner.progressGauge.SetRange(len(scheduledRows)) + self.owner.progressGauge.SetValue(processedCounter) + self.owner.progressGauge.Show() + + if not self.export: + + # if Collection column spectifed check that all collection names resolve to collectionIDs + if collectionColumn != -1: + for rowCounter in scheduledRows: + collectionName = self.csvArray[rowCounter][collectionColumn] + if collectionName.strip() != "" and collectionName not in self.collectionIDs.keys(): + raise Exception,"Unknown collection '%s' at row %s" % (collectionName, rowCounter) + + rowCounter = 0 + for rowCounter in scheduledRows: + if self.StopProcessing: + break + + if rowCounter < len(self.csvArray): + self.Skip = False + processedCounter += 1 + + self.echo("---") + + self.tryPausing("[Paused]") + + # update UI and log + wx.GetApp().Yield() + owner.mainStatusBar.SetStatusText("Processing row %s [%s of %s]" % (rowCounter, processedCounter, rowsToBeProcessedCount), 0) + + action = "Processing item..." + if testOnly: + action = "Validating item..." + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + " Row %s [%s of %s]: %s" % (rowCounter, processedCounter, rowsToBeProcessedCount, action)) + + # process row + savedItemID, savedItemVersion, sourceIdentifier, rowData, rowError = self.processRow(rowCounter, + self.csvArray[rowCounter], + self.tle, + itemdefuuid, + self.collectionIDs, + testOnly, + sourceIdentifierColumn, + targetIdentifierColumn, + targetVersionColumn, + commandOptionsColumn, + attachmentLocationsColumn, + urlsColumn, + resourcesColumn, + customAttachmentsColumn, + collectionColumn, + thumbnailsColumn, + selectedThumbnailColumn, + ownerColumn, + collaboratorsColumn) + + # add to row receipts + rowReceipts[rowCounter] = (savedItemID, savedItemVersion) + if sourceIdentifierColumn != -1: + self.sourceIdentifierReceipts[sourceIdentifier] = (savedItemID, savedItemVersion) + + + # update row in CSV array for receipt and script-processed row data + if itemIdColumn != -1: + # assign itemID to receipt cell + rowData[itemIdColumn] = savedItemID + rowData[versionColumn] = str(savedItemVersion) + if rowErrorColumn != -1: + rowData[rowErrorColumn] = rowError + self.csvArray[rowCounter] = rowData + + # update progress bar + self.owner.progressGauge.SetValue(processedCounter) + + if self.StopProcessing: + self.echo("---") + if self.processingStoppedByScript: + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "Processing halted") + else: + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "Processing halted by user") + + self.echo("---") + + # run End Script + if self.endScript.strip() != "": + try: + exec self.endScript in { + "IMPORT":0, + "EXPORT":1, + "mode":action, + "vars":self.scriptVariables, + "rowCounter":rowCounter, + "testOnly": testOnly, + "institutionUrl":tle.institutionUrl, + "collection":self.collection, + "csvFilePath":self.csvFilePath, + "username":self.username, + "logger":self.logger, + "columnHeadings":self.columnHeadings, + "columnSettings":self.currentColumns, + "successCount":self.successCount, + "errorCount":self.errorCount, + "process":self.process, + "basepath":self.absoluteAttachmentsBasepath, + "sourceIdentifierIndex":sourceIdentifierColumn, + "targetIdentifierIndex":targetIdentifierColumn, + "targetVersionIndex":targetVersionColumn, + "csvData":self.csvArray, + "ebi":self.ebiScriptObject, + "equella":tle, + } + except: + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + scriptErrorMsg = "An error occured in the End Script:\n" + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + scriptErrorMsg, style=2) + else: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + formattedException = "".join(traceback.format_exception_only(exceptionType, exceptionValue))[:-1] + scriptErrorMsg = "An error occured in the End Script:\n%s (line %s)" % (formattedException, traceback.extract_tb(exceptionTraceback)[-1][1]) + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + scriptErrorMsg, style=2) + + + + # output receipts if Item ID column specified and real run + if itemIdColumn != -1: + + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "Writing receipts file...") + + # create receipts folder if one doesn't exist + if not os.path.exists(self.receiptFolder): + os.makedirs(self.receiptFolder) + + # open csv writer and output orginal csv rows using self.columnHeadings as first row (instead of first row of self.csvArray) + f = open(receiptFilename, "wb") + writer = UnicodeWriter(f, self.encoding) + writer.writerow(list(self.columnHeadings)) + for i in range(1, len(self.csvArray)): + writer.writerow(list(self.csvArray[i])) + f.close() + + else: + # export + actionString = "" + if sourceIdentifierColumn == -1 and targetIdentifierColumn == -1: + if self.includeNonLive: + actionString = "Options -> Include non-live items in export" + if actionString != "": + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + actionString) + + self.exportedFiles = [] + self.exportCSV(owner, + self.tle, + itemdefuuid, + self.collectionIDs, + testOnly, + scheduledRows, + sourceIdentifierColumn, + targetIdentifierColumn, + targetVersionColumn, + commandOptionsColumn, + attachmentLocationsColumn, + collectionColumn, + rowsToBeProcessedCount) + + + # form outcome report + errorReport = "" + if self.errorCount > 0: + errorReport = " errors: %s" % (self.errorCount) + resultReport = "Processing complete (success: %s%s)" % (self.successCount, errorReport) + + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + resultReport) + + owner.mainStatusBar.SetStatusText(resultReport, 0) + + + + # getEquellaResourceDetail() retrieves an item or item attachment details necessary for forming an EQUELLA resource-type attachment + def getEquellaResourceDetail(self, resourceUrl, itemdefuuid, collectionIDs, sourceIdentifierColumn, isCalHolding): + + # break up resource url + resourceUrlParts = [] + if resourceUrl[0] == "[" or resourceUrl[0] == "{": + if resourceUrl[-1] == "}": + resourceUrlParts.append(resourceUrl) + elif resourceUrl[-2:-1] == "}/": + resourceUrlParts.append(resourceUrl[:-1]) + else: + resourceUrlParts.append(resourceUrl.split("}/")[0] + "}") + resourceUrlParts += resourceUrl.split("}/")[1].split("/") + else: + resourceUrlParts = resourceUrl.split("/") + + # get item UUID + resourceItemUuid = resourceUrlParts[0] + + # check if itemUUID is actually a sourceIdentifier + # format is {} or []{} or [||]{} + collection = "" + sourceIdentifier = "" + sourceIdentifierXpath = "" + + # no collection specified so use same collection as item + if resourceItemUuid[0] == "{" and resourceItemUuid[-1] == "}": + sourceIdentifier = resourceItemUuid[1:-1] + + # collection specified so resolve to colleciton ID and use that + if resourceItemUuid[0] == "[" and resourceItemUuid[-1] == "}": + collSplitPoint = resourceItemUuid.find("]{") + if collSplitPoint != -1: + sourceIdentifier = resourceItemUuid[collSplitPoint + 2:-1] + collection = resourceItemUuid[1:collSplitPoint] + + # extract an xpath and a source identifier xpath (if one supplied) + collectionParts = collection.split("][") + collection = collectionParts[0] + if len(collectionParts) == 2: + sourceIdentifierXpath = collectionParts[1] + + # get collection ID for collection + if collection in collectionIDs: + itemdefuuid = collectionIDs[collection] + else: + raise Exception, "Collection specified not found: " + collection + + # if source identifer (and optionally collection) specified then find resource by that + if sourceIdentifier != "": + + # first try checking any source identifiers that were processed in the run + if collection == "" and sourceIdentifier in self.sourceIdentifierReceipts: + resourceItemUuid = self.sourceIdentifierReceipts[sourceIdentifier][0] + + # if source identifer not processed in this run then look it up in EQUELLA + else: + if self.debug: + self.echo(" Source identifier = " + sourceIdentifier) + + if sourceIdentifierXpath == "": + if sourceIdentifierColumn != -1: + + # determine source identifier xpath if not specified in resource URL + sourceIdentifierXpath = "/xml/" + self.columnHeadings[sourceIdentifierColumn] + + else: + raise Exception, "No source identifier specified." + + searchFilter = sourceIdentifierXpath + "='" + sourceIdentifier + "'" + results = self.tle.search(0, 10, '/item/name', [itemdefuuid], searchFilter, query='') + + # if any matches get first matching item for editing + if int(results.getNode("available")) > 0: + resourceItemUuid = results.getNode("result/xml/item/@id") + if self.debug: + self.echo(" Resource item found by source identifier = '%s' in collectionID = '%s' (%s)" % (sourceIdentifier, itemdefuuid, resourceItemUuid)) + else: + raise Exception, "Item not found with source identifier '%s'" % sourceIdentifier + + + # get item version + if len(resourceUrlParts) == 1: + resourceItemVersion = 0 + elif resourceUrlParts[1] == "": + resourceItemVersion = 0 + else: + resourceItemVersion = int(resourceUrlParts[1]) + + # get attachment path if any + attachmentPath = "" + if len(resourceUrlParts) > 2: + attachmentPath = "/".join(resourceUrlParts[2:]) + + # retrieve item XML and get attachment UUID and description + resourceXml= self.tle.getItem(resourceItemUuid, resourceItemVersion) + resourceAttachmentUuid = "" + if attachmentPath == "": + # resource is item itself + resourceName = resourceXml.getNode("item/name") + + # if resource is CAL holding then use explicit item version + if isCalHolding: + resourceItemVersion = resourceXml.getNode("item/@version") + else: + if attachmentPath.upper() == "": + # get package details + # try for SCORM package + for attachmentSubtree in resourceXml.iterate("item/attachments/attachment"): + if attachmentSubtree.getNode("@type") == "custom" and attachmentSubtree.getNode("type") == "scorm": + resourceAttachmentUuid = attachmentSubtree.getNode("uuid") + resourceName = attachmentSubtree.getNode("description") + break + if resourceAttachmentUuid == None or resourceAttachmentUuid == "": + # no SCORM package so try for IMS package + resourceAttachmentUuid = resourceXml.getNode("item/itembody/packagefile/@uuid") + resourceName = resourceXml.getNode("item/itembody/packagefile") + if resourceAttachmentUuid == None: + raise Exception, "package not found in item %s/%s" % (resourceItemUuid, resourceItemVersion) + else: + # resource is a file/url attachment + for attachmentSubtree in resourceXml.iterate("item/attachments/attachment"): + if attachmentSubtree.getNode("file") == attachmentPath: + resourceAttachmentUuid = attachmentSubtree.getNode("uuid") + resourceName = attachmentSubtree.getNode("description") + break + if resourceAttachmentUuid == "": + raise Exception, "%s not found in item %s/%s" % (attachmentPath, resourceItemUuid, resourceItemVersion) + + return resourceItemUuid, resourceItemVersion, resourceAttachmentUuid, resourceName + + # addCALRelations() adds CAL holding relations to a CAL portion item + def addCALRelations(self, holdingMetadataTarget, itemXml): + holdingAttachmentUUIDs = itemXml.getNodes(holdingMetadataTarget) + if len(holdingAttachmentUUIDs) > 0: + holdingAttachmentFound = False + for attachment in itemXml.iterate("item/attachments/attachment"): + if attachment.getNode("uuid") == holdingAttachmentUUIDs[0]: + relation = itemXml.newSubtree("item/relations/targets/relation") + relation.createNode("@resource", attachment.getNode("uuid")) + relation.createNode("@type", "CAL_HOLDING") + relationitem = relation.newSubtree("item") + relationitem.createNode("name", attachment.getNode("description")) + + holdingUuid = "" + holdingVersion = "" + for attachmentAttribute in attachment.iterate("attributes/entry"): + entryName = attachmentAttribute.getNode("string") + if entryName == "uuid": + holdingUuid = attachmentAttribute.getNodes("string")[1] + if entryName == "version": + holdingVersion = attachmentAttribute.getNode("int") + + relationitem.createNode("@uuid", holdingUuid) + relationitem.createNode("@version", holdingVersion) + holdingAttachmentFound = True + break + if not holdingAttachmentFound: + raise Exception, "No holding item attached to this portion" + else: + raise Exception, "No metadata targets for holding items found in this portion" + + + # processRow() creates a single item in EQUELLA based on one row of metadata + def processRow(self, + rowCounter, + meta, + tle, + itemdefuuid, + collectionIDs, + testOnly, + sourceIdentifierColumn, + targetIdentifierColumn, + targetVersionColumn, + commandOptionsColumn, + attachmentLocationsColumn, + urlsColumn, + resourcesColumn, + customAttachmentsColumn, + collectionColumn, + thumbnailsColumn, + selectedThumbnailColumn, + ownerColumn, + collaboratorsColumn): + + + # unzipAttachment attachs and unzips an attachment and sets start pages based on tuple of tuples formatted attachment name + def unzipAttachment(): + self.echo(" Unzip file") + if not testOnly: + attemptingUpload = True + item.attachFile('_zips/' + filename, file(filepath, "rb"), uploadStatus, self.chunkSize) + if self.StopProcessing: + return + attemptingUpload = False + wx.GetApp().Yield() + item.unzipFile('_zips/' + filename, filename) + + if attachmentLinkName != "": + # + try: + # read the attachment link name in as a tuple of tuples using exec() + # add a superfluous tuple to coax the string into a tuple of tuples even if only one tuple + startPagesListAsString = "((\"#####\",\"#####\")," + attachmentLinkName[1:] + exec "startPagesList = " + startPagesListAsString in globals(), locals() + + except: + raise Exception, "List of links to unzipped files incorrectly formatted." + + # populate a dictionary rendition of the Start Pages List + startPagesDict = {} + for startPage in startPagesList: + if startPage[0] != "#####": + startPagesDict[startPage[0]] = startPage[1] + + # iterate through Start Pages List + for startPage in startPagesList: + + # check if start page for zip itself + if startPage[0] == filename: + + # generate attachment UUID + attachmentUUID = str(uuid.uuid4()) + + # add attachment element + item.addStartPage(startPage[1], "_zips/" + filename, filesize, attachmentUUID) + + # add corresponding metadata target for attachment + if self.attachmentMetadataTargets: + item.getXml().createNode(self.columnHeadings[n], attachmentUUID) + + + # check if wildcard exists + elif startPage[0] == "*": + for archiveFile in zfobj.namelist(): + + # check if file not already specified, not zip itself and not a folder + if (archiveFile not in startPagesDict) and (archiveFile != filename) and not archiveFile.endswith("/"): + # get file size + archiveFilesize = zfobj.getinfo(archiveFile).file_size + + # generate attachment UUID + attachmentUUID = str(uuid.uuid4()) + + # add attachment element + item.addStartPage(os.path.basename(archiveFile), filename + "/" + archiveFile, archiveFilesize, attachmentUUID) + + # add corresponding metadata target for attachment + if self.attachmentMetadataTargets: + item.getXml().createNode(self.columnHeadings[n], attachmentUUID) + + + # not the zip itself or a wildcard so name as specified + elif startPage[0] in startPagesDict: + # get file size + archiveFilesize = zfobj.getinfo(startPage[0]).file_size + + # generate attachment UUID + attachmentUUID = str(uuid.uuid4()) + + # add attachment element + item.addStartPage(startPage[1], filename + "/" + startPage[0], archiveFilesize, attachmentUUID) + + # add corresponding metadata target for attachment + if self.attachmentMetadataTargets: + item.getXml().createNode(self.columnHeadings[n], attachmentUUID) + + + failCount = 0 + retriesDone = False + + # loop for retrying if network errors occur + while not retriesDone: + try: + + wx.GetApp().Yield() + createNewItem = True + createNewVersion = False + allRowsError = False + itemID = "nil" + itemVersion = 0 + savedItemID = "" + savedItemVersion = "" + n = -1 + attemptingUpload = False + imsmanifest = None + + if self.Skip: + self.echo(" Row skipped") + return "", "", "", [] + + # resolve owner to an ID + ownerUsername = meta[ownerColumn].strip() + ownerID = "" + if ownerColumn != -1 and ownerUsername != "": + self.echo(" Owner: " + ownerUsername) + try: + matchingUsers = self.tle.searchUsersByGroup("", ownerUsername) + except: + error = str(sys.exc_info()[1]) + if error.rfind('HTTP') != -1 and error.rfind('404') != -1: + self.echo(" ERROR: Cannot use Owner or Collaborators column datatypes with this version of EQUELLA", style=2) + allRowsError = True + raise Exception, error + + matchingUserNodes = matchingUsers.getNodes("user", False) + + # if any matches get first matching user + if len(matchingUserNodes) > 0: + ownerID = matchingUserNodes[0].getElementsByTagName("uuid")[0].firstChild.nodeValue + elif self.useEBIUsername: + ownerColumn = -1 + self.echo(" '%s' not found so ignoring." % (ownerUsername, self.username)) + else: + if self.saveNonexistentUsernamesAsIDs: + ownerID = ownerUsername + else: + raise Exception, "'%s' not found so cannot set owner." % ownerUsername + + # resolve collaborators to IDs + collaboratorIDs = [] + if collaboratorsColumn != -1 and meta[collaboratorsColumn].strip() != "": + # check if this column is a multi-value column + if self.currentColumns[collaboratorsColumn][self.COLUMN_DELIMITER].strip() != "": + # column is a multi-value column + actualDelimiter = self.currentColumns[collaboratorsColumn][self.COLUMN_DELIMITER].strip() + else: + # column is NOT a multi-value column + actualDelimiter = "@~@~@~@~@~@~@" + + specifiedCollaborators = meta[collaboratorsColumn].split(actualDelimiter) + self.echo(" Collaborators: " + ",".join(specifiedCollaborators)) + for specifiedCollaborator in specifiedCollaborators: + try: + matchingUsers = self.tle.searchUsersByGroup("", specifiedCollaborator.strip()) + except: + error = str(sys.exc_info()[1]) + if error.rfind('HTTP') != -1 and error.rfind('404') != -1: + self.echo(" ERROR: Cannot use Owner or Collaborators column datatypes with this version of EQUELLA", style=2) + allRowsError = True + raise Exception, error + matchingUserNodes = matchingUsers.getNodes("user", False) + + # if any matches get first matching user + if len(matchingUserNodes) > 0: + collaboratorID = matchingUserNodes[0].getElementsByTagName("uuid")[0].firstChild.nodeValue + collaboratorIDs.append(collaboratorID) + elif self.ignoreNonexistentCollaborators: + self.echo(" '%s' not found so ignoring that collaborator." % specifiedCollaborator.strip()) + else: + if self.saveNonexistentUsernamesAsIDs: + collaboratorIDs.append(specifiedCollaborator) + else: + raise Exception, "'%s' not found so cannot set collaborators." % specifiedCollaborator.strip() + + # get command options + commandOptions = [] + if commandOptionsColumn != -1: + + # get position of command options column + tempCommandOptions = [commandOption.strip().upper() for commandOption in meta[commandOptionsColumn].split(",")] + for commandOption in tempCommandOptions: + if commandOption != "": + commandOptions.append(commandOption) + if len(commandOptions) > 0: + self.echo(" Command options: " + ",".join(commandOptions)) + + # get targeted item version if target version specified + if targetVersionColumn != -1 and meta[targetVersionColumn].strip() != "": + try: + itemVersion = int(meta[targetVersionColumn].strip()) + if itemVersion < -1: + raise Exception, "Invalid item version specified" + except: + raise Exception, "Invalid item version specified" + + # if Source Identifier column specified check if item exists by sourceIdentifier + if sourceIdentifierColumn != -1: + sourceIdentifier = meta[sourceIdentifierColumn].strip() + self.echo(" Source identifier = " + sourceIdentifier) + if sourceIdentifier.find("'") != -1: + raise Exception, "Source identifier cannot contain apostrophes" + if targetVersionColumn != -1 and meta[targetVersionColumn].strip() != "": + self.echo(" Target version = " + meta[targetVersionColumn].strip()) + searchFilter = "/xml/" + self.columnHeadings[sourceIdentifierColumn] + "='" + sourceIdentifier + "'" + + if itemVersion != 0: + onlyLive = False + limit = 50 + else: + onlyLive = True + limit = 1 + + results = tle.search(0, limit, '/item/name', [itemdefuuid], searchFilter, query='', onlyLive=onlyLive) + + # if any matches get first matching item for editing + if int(results.getNode("available")) > 0: + + if itemVersion == 0: + # get first live version + itemID = results.getNode("result/xml/item/@id") + itemVersion = results.getNode("result/xml/item/@version") + else: + if itemVersion != -1: + # find item by item version + itemFound = False + for itemResult in results.iterate("result"): + if itemResult.getNode("xml/item/@version") == str(itemVersion): + itemID = itemResult.getNode("xml/item/@id") + itemFound = True + break + if not itemFound: + raise Exception, "Item not found" + else: + # find item with highest version + highestVersionFound = 0 + for itemResult in results.iterate("result"): + if int(itemResult.getNode("xml/item/@version")) > highestVersionFound: + highestVersionFound = int(itemResult.getNode("xml/item/@version")) + itemID = itemResult.getNode("xml/item/@id") + itemVersion = highestVersionFound + + self.echo(" Item exists in EQUELLA (" + itemID + "/" + str(itemVersion) + ")") + createNewItem = False + else: + if itemVersion == 0 or itemVersion == -1: + self.echo(" Item not found") + else: + raise Exception, "Item not found" + else: + # if Target Identifier column specified edit item by ID (using latest version of item) + if targetIdentifierColumn != -1 and meta[targetIdentifierColumn].strip() != "": + itemID = meta[targetIdentifierColumn].strip() + + self.echo(" Target identifier = " + itemID) + if targetVersionColumn != -1 and meta[targetVersionColumn].strip() != "": + self.echo(" Target version = " + meta[targetVersionColumn].strip()) + + # try getting item + foundItem = tle.getItem(itemID, itemVersion) + + self.echo(" Item exists in EQUELLA (" + itemID + "/" + foundItem.getNode("item/@version") + ")") + createNewItem = False + + if "DELETE" in commandOptions: + # check that if using target identifiers is specified that this row has one + if targetIdentifierColumn != -1 and meta[targetIdentifierColumn].strip() == "" and sourceIdentifierColumn == -1: + raise Exception, "Neither source identifer nor target identifier supplied" + + # run Row Pre-Script + if self.preScript.strip() != "": + try: + exec self.preScript in { + "IMPORT":0, + "EXPORT":1, + "NEWITEM":0, + "NEWVERSION":1, + "EDITITEM":2, + "DELETEITEM":3, + "mode":0, + "action":3, + "vars":self.scriptVariables, + "rowData":meta, + "rowCounter":rowCounter, + "testOnly": testOnly, + "institutionUrl":tle.institutionUrl, + "collection":self.collection, + "csvFilePath":self.csvFilePath, + "username":self.username, + "logger":self.logger, + "columnHeadings":self.columnHeadings, + "columnSettings":self.currentColumns, + "successCount":self.successCount, + "errorCount":self.errorCount, + "process":self.process, + "basepath":self.absoluteAttachmentsBasepath, + "sourceIdentifierIndex":sourceIdentifierColumn, + "targetIdentifierIndex":targetIdentifierColumn, + "targetVersionIndex":targetVersionColumn, + "csvData":self.csvArray, + "ebi":self.ebiScriptObject, + "equella":tle, + } + except: + if self.debug: + raise + else: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + formattedException = "".join(traceback.format_exception_only(exceptionType, exceptionValue))[:-1] + scriptErrorMsg = "An error occured in the Row Pre-Script:\n%s (line %s)" % (formattedException, traceback.extract_tb(exceptionTraceback)[-1][1]) + raise Exception, scriptErrorMsg + + if not createNewItem: + if not testOnly: + # delete existing item + tle._forceUnlock(itemID, itemVersion) + tle._deleteItem(itemID, itemVersion) + self.echo(" Item successfully deleted") + + savedItemID = itemID + savedItemVersion = itemVersion + else: + self.echo(" Item valid to delete") + + self.successCount += 1 + + else: + scriptAction = 0 + + # create new item or prepare existing one + if createNewItem: + # get collection ID + collectionID = itemdefuuid + + # override the collection ID if one has been specified in the row + if collectionColumn != -1: + collectionName = meta[collectionColumn].strip() + if collectionName != "": + collectionID = collectionIDs[collectionName] + self.echo(" Target collection: '%s'" % collectionName) + + item = tle.createNewItem(collectionID) + itemID = item.uuid + itemVersion = item.version + item.prop.setNode("item/thumbnail", "default") + else: + if self.createNewVersions or "VERSION" in commandOptions: + # create new version + item = tle.newVersionItem(itemID, itemVersion) + createNewVersion = True + scriptAction = 1 + self.echo(" Creating new version") + else: + # open existing version for editing + tle._forceUnlock(itemID, itemVersion) + item = tle.editItem(itemID, itemVersion, 'true') + scriptAction = 2 + self.echo(" Editing item") + + if collectionColumn != -1 and meta[collectionColumn].strip() != "": + self.echo(" Target collection: '%s'. Cannot use target collection for existing items (ignoring)" % meta[collectionColumn].strip()) + + # delete all attachments if hyperlinks column or attachments column specified + if (urlsColumn != -1 or attachmentLocationsColumn != -1 or resourcesColumn != -1 or customAttachmentsColumn != -1) and \ + not self.appendAttachments and not 'APPENDATTACH' in commandOptions: + + # delete attachments + item.deleteAttachments() + + # remove any possible package link and navigation nodes + item.prop.removeNode("item/itembody/packagefile") + item.prop.removeNode("item/navigationNodes/node") + + # delete all existing custom metadata + if self.existingMetadataMode not in [self.REPLACEMETA, self.APPENDMETA] and 'APPENDMETA' not in commandOptions and 'REPLACEMETA' not in commandOptions: + for childNode in item.prop.root.childNodes: + if childNode.nodeName == "item": + for itemChildNode in childNode.childNodes: + if itemChildNode.nodeName not in self.itemSystemNodes: + childNode.removeChild(itemChildNode) + else: + item.prop.root.removeChild(childNode) + else: + # appending metadata but still delete source identifier node to avoid duplication + if sourceIdentifierColumn != -1: + # check that it is not an attribute + if "@" not in self.columnHeadings[sourceIdentifierColumn]: + item.getXml().removeNode(self.columnHeadings[sourceIdentifierColumn]) + + # run Row Pre-Script + if self.preScript.strip() != "": + try: + exec self.preScript in { + "IMPORT":0, + "EXPORT":1, + "NEWITEM":0, + "NEWVERSION":1, + "EDITITEM":2, + "DELETEITEM":3, + "mode":0, + "action":scriptAction, + "vars":self.scriptVariables, + "rowData":meta, + "rowCounter":rowCounter, + "testOnly": testOnly, + "institutionUrl":tle.institutionUrl, + "collection":self.collection, + "csvFilePath":self.csvFilePath, + "username":self.username, + "logger":self.logger, + "itemId":itemID, + "itemVersion":itemVersion, + "xml":item.prop, + "xmldom":item.newDom, + "columnHeadings":self.columnHeadings, + "columnSettings":self.currentColumns, + "successCount":self.successCount, + "errorCount":self.errorCount, + "process":self.process, + "basepath":self.absoluteAttachmentsBasepath, + "sourceIdentifierIndex":sourceIdentifierColumn, + "targetIdentifierIndex":targetIdentifierColumn, + "csvData":self.csvArray, + "ebi":self.ebiScriptObject, + "equella":tle, + } + except: + if self.debug: + raise + else: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + formattedException = "".join(traceback.format_exception_only(exceptionType, exceptionValue))[:-1] + scriptErrorMsg = "An error occured in the Row Pre-Script:\n%s (line %s)" % (formattedException, traceback.extract_tb(exceptionTraceback)[-1][1]) + raise Exception, scriptErrorMsg + + + + # process the contents of attachmentMetadataColumn for any {uuid) placeholders + attachmentUUIDPlaceholders = {} + + hyperlinkColumnCount = 0 + attachmentColumnCount = 0 + equellaResourceColumnCount = 0 + calHoldingMetadataTarget = "" + thumbnailSelected = False + + # get thumbnail settings for row if any + thumbnails = [] + selectedThumbnail = "" + if thumbnailsColumn != -1: + thumbnailDelimiter = self.currentColumns[thumbnailsColumn][self.COLUMN_DELIMITER].strip() + if thumbnailDelimiter != "": + thumbnails = meta[thumbnailsColumn].split(thumbnailDelimiter) + thumbnails = [thumb.strip() for thumb in thumbnails] + else: + thumbnails = meta[thumbnailsColumn].strip() + if selectedThumbnailColumn != -1: + selectedThumbnail = meta[selectedThumbnailColumn].strip() + + # iterate through the columns of the CSV row + for n in range(0, len(meta)): + wx.GetApp().Yield() + if self.StopProcessing: + break + + isMetadataField = True + + # check if this column is a multi-value column + if self.currentColumns[n][self.COLUMN_DELIMITER].strip() != "": + # column is a multi-value column + actualDelimiter = self.currentColumns[n][self.COLUMN_DELIMITER].strip() + else: + # column is NOT a multi-value column + actualDelimiter = "@~@~@~@~@~@~@" + + + # add hyperlink from csv + if self.currentColumns[n][self.COLUMN_DATATYPE] == self.URLS: + + hyperlinkColumnCount += 1 + + isMetadataField = False + + # delete all url metadata targets if first occurence + if self.attachmentMetadataTargets: + if self.columnHeadings[:n].count(self.columnHeadings[n]) == 0: + item.getXml().removeNode(self.columnHeadings[n]) + + # split for multi-value field + values = meta[n].split(actualDelimiter) + for i in range(len(values)): + if values[i].strip() != "": + url = unicode(values[i].replace(" ", "%20")) + + self.echo(" Hyperlink: " + url) + + # find corresponding Hyperlink Name column for URL Location column + hyperlinkName = "" + hyperlinkNameColumnCount = 0 + for col in range(0, len(meta)): + if self.currentColumns[col][self.COLUMN_DATATYPE] == self.HYPERLINKNAMES: + + hyperlinkNameColumnCount += 1 + + if hyperlinkNameColumnCount == hyperlinkColumnCount: + # split for multi-value field + names = meta[col].split(actualDelimiter) + + # match name[i] to value[i] but check for array out of bounds + if i < len(names): + if names[i].strip() != "": + # add start page with attachment name + hyperlinkName = names[i] + break + + # generate uuid for URL's attachment element + attachmentUUID = str(uuid.uuid4()) + + # add start page after checking if a hyperlink name has been specified + if hyperlinkName != "": + item.addUrl(hyperlinkName, url, attachmentUUID) + else: + # add start page WITHOUT hyperlink name + item.addUrl(url, url, attachmentUUID) + + # add metadata target for remote attachment + if self.attachmentMetadataTargets: + if self.columnHeadings[n].strip() != "" and self.columnHeadings[n][:1] != "#": + item.getXml().createNode(self.columnHeadings[n], attachmentUUID) + + # add attachment from csv + if self.currentColumns[n][self.COLUMN_DATATYPE] == self.ATTACHMENTLOCATIONS: + + attachmentColumnCount += 1 + + isMetadataField = False + + # delete all attachment metadata targets if this is the first occurence + if self.attachmentMetadataTargets: + if self.columnHeadings[:n].count(self.columnHeadings[n]) == 0: + item.getXml().removeNode(self.columnHeadings[n]) + + # split for multi-value field + values = meta[n].split(actualDelimiter) + for i in range(len(values)): + if values[i].strip() != "": + + # get absolute path to file + filepath = os.path.join(self.absoluteAttachmentsBasepath, values[i]) + + # ensure that attachment specified is not a directory + if os.path.isdir(filepath): + raise Exception, filepath + " is not a file" + + # get filename and file size + filename = os.path.basename(filepath) + filesize = os.path.getsize(filepath) + + uploadStatus = " " + + # find corresponding Attachment Name column for Attachment Location column + attachmentLinkName = "" + attachmentNameColumnCount = 0 + for col in range(0, len(meta)): + if self.currentColumns[col][self.COLUMN_DATATYPE] == self.ATTACHMENTNAMES: + + attachmentNameColumnCount += 1 + + if attachmentNameColumnCount == attachmentColumnCount: + # split for multi-value field + names = meta[col].split(actualDelimiter) + + # match name[i] to value[i] but check for array out of bounds + if i < len(names): + if names[i].strip() != "": + # add start page with attachment name + attachmentLinkName = names[i] + break + + # echo out attachment information including start page link + attachmentLinkNameDisplay = "" + if self.debug and attachmentLinkName != "": + attachmentLinkNameDisplay = ' -> "' + attachmentLinkName + '"' + filesizeDisplay = self.group(filesize) + if filesize > 999999 and not self.debug: + filesizeDisplay = filesizeDisplay[:-8] + "." + filesizeDisplay[-7:-5] + " MB" + else: + filesizeDisplay += " bytes" + self.echo(" Attachment: " + filename + " (" + filesizeDisplay + ")" + attachmentLinkNameDisplay) + + # process command options + attachmentType = self.attachmentTypeFile + if ("UNZIP" in commandOptions or "IMS" in commandOptions or "SCORM" in commandOptions or "AUTO" in commandOptions): + + # check if zip file by extension + if (os.path.splitext(filename)[1].upper() == ".ZIP"): + self.echo(" Attachment is a zip file") + zfobj = zipfile.ZipFile(filepath) + + # check if commands are either IMS, SCORM or AUTO + if ("IMS" in commandOptions or "SCORM" in commandOptions or "AUTO" in commandOptions): + if ("imsmanifest.xml" in zfobj.namelist()): + imsmanifest = PropBagEx(zfobj.read("imsmanifest.xml").decode("utf8")) + self.echo(" IMS manifest found, treat as IMS package") + + # generate UUID for IMS package + attachmentUUID = str(uuid.uuid4()) + + # add metadata target for attachment + if self.attachmentMetadataTargets: + + # only add an attachment metadata target if the column is neither blank nor starts with a "#" + if self.columnHeadings[n].strip() != "" and self.columnHeadings[n][:1] != "#": + item.getXml().createNode(self.columnHeadings[n], attachmentUUID) + + # use filename for IMS/SCORM attachment's description node if no Attachment Name supplied + if attachmentLinkName == "": + attachmentLinkName = filename + + if ("IMS" in commandOptions or "AUTO" in commandOptions): + + # attach IMS package + attemptingUpload = True + + # if imsmanifest indicates that package is a SCORM package then attach as a SCORM package + if self.scormformatsupport and imsmanifest.getNode("metadata/schema") == "ADL SCORM": + self.echo(" Package is a SCORM package") + item.attachSCORM(file(filepath, "rb"), filename, attachmentLinkName, uploadStatus, not testOnly, filesize, attachmentUUID, self.chunkSize) + else: + item.attachIMS(file(filepath, "rb"), filename, attachmentLinkName, uploadStatus, not testOnly, filesize, attachmentUUID, self.chunkSize) + attemptingUpload = False + + attachmentType = self.attachmentTypeIMS + + if "SCORM" in commandOptions: + attemptingUpload = True + item.attachSCORM(file(filepath, "rb"), filename, attachmentLinkName, uploadStatus, not testOnly, filesize, attachmentUUID, self.chunkSize) + attemptingUpload = False + + attachmentType = self.attachmentTypeSCORM + + # check if command is AUTO + elif ("AUTO" in commandOptions): + self.echo(" No IMS manifest found, treat as simple zip file") + + # unzip file + unzipAttachment() + attachmentType = self.attachmentTypeZip + + # check if command is IMS + elif ("IMS" in commandOptions or "SCORM" in commandOptions): + raise Exception, "No IMS manifest found, cannot use IMS or SCORM command option" + + # check if command is UNZIP + elif ("UNZIP" in commandOptions): + # unzip file + attemptingUpload = True + unzipAttachment() + attemptingUpload = False + attachmentType = self.attachmentTypeZip + + # file is not a zip file, check if command is AUTO + elif ("AUTO" in commandOptions): + if not testOnly: + attemptingUpload = True + item.attachFile(filename, file(filepath, "rb"), uploadStatus, self.chunkSize) + attemptingUpload = False + + attachmentType = self.attachmentTypeFile + + # file is not a zip file but UNZIP command requested + elif ("UNZIP" in commandOptions): + raise Exception, "Not a zip file, cannot use UNZIP or IMS command options" + else: + # no command options + if not testOnly: + attemptingUpload = True + item.attachFile(filename, file(filepath, "rb"), uploadStatus, self.chunkSize) + attemptingUpload = False + + attachmentType = self.attachmentTypeFile + + # add start page if attachment is a simple file + if attachmentType == self.attachmentTypeFile: + + # generate uuid for attachment + attachmentUUID = str(uuid.uuid4()) + + # determine if attachment should have thumbnails suppressed + thumbnail = "" + if thumbnailsColumn != -1 and values[i].strip() not in thumbnails: + thumbnail = "suppress" + + # check if custom thumbnail has been specified + for thumb in thumbnails: + thumbparts = thumb.split(":") + if len(thumbparts) == 2: + if thumbparts[0].strip() == values[i].strip(): + thumbnail = thumbparts[1].strip() + + # check if file extension matches a thumbnail wildcard + if "*" + os.path.splitext(filepath)[1].lower() in (wildcard.lower() for wildcard in thumbnails): + thumbnail = "" + + if attachmentLinkName != "": + # add start page with provided attachment link name + item.addStartPage(attachmentLinkName, filename, filesize, attachmentUUID, thumbnail) + else: + # add start page using filename as attachment link + item.addStartPage(filename, filename, filesize, attachmentUUID, thumbnail) + + # add metadata target for attachment + if self.attachmentMetadataTargets: + # only add an attachment metadata target if the column is neither blank nor starts with a "#" + if self.columnHeadings[n].strip() != "" and self.columnHeadings[n][:1] != "#": + item.getXml().createNode(self.columnHeadings[n], attachmentUUID) + + # check if attachment is selected thumbnail + if values[i].strip() == selectedThumbnail: + item.getXml().setNode("item/thumbnail", "custom:" + attachmentUUID) + thumbnailSelected = True + + # check if attachment extension matches selected thumbnail wildcard + elif thumbnail != "suppress": + if not thumbnailSelected: + if "*" + os.path.splitext(filepath)[1].lower() == selectedThumbnail: + item.getXml().setNode("item/thumbnail", "custom:" + attachmentUUID) + thumbnailSelected = True + + # add raw files from csv + if self.currentColumns[n][self.COLUMN_DATATYPE] == self.RAWFILES: + + attachmentColumnCount += 1 + + isMetadataField = False + + # split for multi-value field + values = meta[n].split(actualDelimiter) + for i in range(len(values)): + if values[i].strip() != "": + + uploadStatus = " " + attachIndent = "" + + # find corresponding Attachment Name column for Attachment Location column + attachmentLinkName = "" + attachmentNameColumnCount = 0 + for col in range(0, len(meta)): + if self.currentColumns[col][self.COLUMN_DATATYPE] == self.ATTACHMENTNAMES: + + attachmentNameColumnCount += 1 + + if attachmentNameColumnCount == attachmentColumnCount: + # split for multi-value field + names = meta[col].split(actualDelimiter) + + # match name[i] to value[i] but check for array out of bounds + if i < len(names): + if names[i].strip() != "": + # add start page with attachment name + attachmentLinkName = names[i] + break + + rawFiles = [] + + # check if a folder rather than a file is specified + if values[i].strip().endswith("*"): + + uploadStatus = " " + attachIndent = " " + prependFolder = "" + targetDisplay = "" + if attachmentLinkName != "": + prependFolder = attachmentLinkName[:-1] + targetDisplay = " -> " + attachmentLinkName + self.echo(" Folder: " + values[i] + targetDisplay) + + # recurse through the folder adding files to be uploaded + rootdir = os.path.join(self.absoluteAttachmentsBasepath, values[i]).strip()[:-2] + for dirname, dirnames, filenames in os.walk(rootdir): + for filename in filenames: + rawFile = {} + rawFile["filepath"] = os.path.join(dirname, filename) + rawFile["originalfilename"] = os.path.relpath(rawFile["filepath"], rootdir) + rawFile["filename"] = prependFolder + rawFile["originalfilename"] + rawFile["filesize"] = os.path.getsize(rawFile["filepath"]) + os.path.relpath(os.path.join(dirname, filename), rootdir) + rawFiles.append(rawFile) + + else: + # a single file was specified so add that as the only file to be uploaded + rawFile = {} + rawFile["filepath"] = os.path.join(self.absoluteAttachmentsBasepath, values[i]) + + # ensure that attachment specified is not a directory + if os.path.isdir(rawFile["filepath"]): + raise Exception, filepath + " is not a file" + + rawFile["filename"] = os.path.basename(rawFile["filepath"]) + rawFile["originalfilename"] = rawFile["filename"] + if attachmentLinkName != "": + rawFile["filename"] = attachmentLinkName + rawFile["filesize"] = os.path.getsize(rawFile["filepath"]) + rawFiles.append(rawFile) + + + # upload all raw files specified + for rawFile in rawFiles: + + # echo out attachment information including start page link + attachmentLinkNameDisplay = "" + if attachmentLinkName != "" and not values[i].strip().endswith("*"): + attachmentLinkNameDisplay = ' -> "' + attachmentLinkName + '"' + filesizeDisplay = self.group(rawFile["filesize"]) + if rawFile["filesize"] > 999999 and not self.debug: + filesizeDisplay = filesizeDisplay[:-8] + "." + filesizeDisplay[-7:-5] + " MB" + else: + filesizeDisplay += " bytes" + self.echo(attachIndent + " Attachment: " + rawFile["originalfilename"] + " (" + filesizeDisplay + ")" + attachmentLinkNameDisplay) + + if not testOnly: + attemptingUpload = True + item.attachFile(rawFile["filename"], file(rawFile["filepath"], "rb"), uploadStatus, self.chunkSize) + attemptingUpload = False + + + + # add EQUELLA resources + if self.currentColumns[n][self.COLUMN_DATATYPE] == self.EQUELLARESOURCES: + + equellaResourceColumnCount += 1 + + isMetadataField = False + + isCalHolding = False + if calHoldingMetadataTarget == "" and "CAL_PORTION" in commandOptions: + isCalHolding = True + calHoldingMetadataTarget = self.columnHeadings[n] + + # delete all attachment metadata targets if this is the first occurence + if self.attachmentMetadataTargets: + if self.columnHeadings[:n].count(self.columnHeadings[n]) == 0: + item.getXml().removeNode(self.columnHeadings[n]) + + # split for multi-value field + values = meta[n].split(actualDelimiter) + for i in range(len(values)): + if values[i].strip() != "": + + resourceUrl = unicode(values[i]) + self.echo(" EQUELLA resource: " + resourceUrl) + + resourceItemUuid, resourceItemVersion, resourceAttachmentUuid, resourceName = self.getEquellaResourceDetail(resourceUrl, itemdefuuid, collectionIDs, sourceIdentifierColumn, isCalHolding) + if self.debug: + self.echo(" resourceItemUuid = " + resourceItemUuid) + self.echo(" resourceItemVersion = " + str(resourceItemVersion)) + self.echo(" resourceAttachmentUuid = " + resourceAttachmentUuid) + self.echo(" resourceName = " + resourceName) + + # find corresponding EQUELLA Resource Name column for URL Location column + equellaResourceNameColumnCount = 0 + for col in range(0, len(meta)): + if self.currentColumns[col][self.COLUMN_DATATYPE] == self.EQUELLARESOURCENAMES: + + equellaResourceNameColumnCount += 1 + + if equellaResourceNameColumnCount == equellaResourceColumnCount: + # split for multi-value field + names = meta[col].split(actualDelimiter) + + # match name[i] to value[i] but check for array out of bounds + if i < len(names): + if names[i].strip() != "": + # add start page with attachment name + resourceName = names[i] + break + + # generate uuid for URL's attachment element + attachmentUUID = str(uuid.uuid4()) + + # add item resource + item.attachResource(resourceItemUuid, resourceItemVersion, resourceName, attachmentUUID, resourceAttachmentUuid) + + # add metadata target for resource + if self.attachmentMetadataTargets: + if self.columnHeadings[n].strip() != "" and self.columnHeadings[n][:1] != "#": + item.getXml().createNode(self.columnHeadings[n], attachmentUUID) + + # process custom attachments + if self.currentColumns[n][self.COLUMN_DATATYPE] == self.CUSTOMATTACHMENTS and meta[n].strip() != "": + + # form XML document from fragment (no root nodes needed in fragment) + xmlAttachmentElementsFragmentString = "%s" % (self.encoding, meta[n].strip()) + xmlAttachmentElements = PropBagEx(xmlAttachmentElementsFragmentString.encode(self.encoding)) + + # check /uuid elements for UUID placeholders + for attachmentElement in xmlAttachmentElements.iterate("attachment"): + + # get attachment UUID if supplied + attachmentUUID = attachmentElement.getNode("uuid") + + # generate an attachment UUID if one is not supplied + if attachmentUUID == None or attachmentUUID == "": + + # generate UUID + attachmentUUID = str(uuid.uuid4()) + + # replace placeholder in /uuid element with generated UUID + attachmentElement.setNode("uuid", attachmentUUID) + + if attachmentElement.getNode("selected_thumbnail") == "true": + item.getXml().setNode("item/thumbnail", "custom:" + attachmentUUID) + thumbnailSelected = True + + # append to system attachments + item.getXml().setNode("item/attachments", "") + item.getXml().root.getElementsByTagName("item")[0].getElementsByTagName("attachments")[0].appendChild(attachmentElement.root.cloneNode(True)) + + # add metadata target + if self.attachmentMetadataTargets: + if self.columnHeadings[n].strip() != "" and self.columnHeadings[n][:1] != "#": + item.getXml().createNode(self.columnHeadings[n], attachmentUUID) + + # process metadata field + if self.currentColumns[n][self.COLUMN_DATATYPE] == self.METADATA: + + # check if column is flagged for XML fragments + if self.currentColumns[n][self.COLUMN_XMLFRAGMENT] == "YES": + + # display to log if necessary + if self.currentColumns[n][self.COLUMN_DISPLAY] == "YES": + self.echo(" %s: %s" % (self.columnHeadings[n], meta[n].strip())) + + if meta[n].strip() != "": + + # form XML document from fragment (no root nodes needed in fragment) + xmlFragmentString = "%s" % (meta[n]) + xmlFragment = PropBagEx(unicode(xmlFragmentString)) + + # remove all indenting + stripNode(xmlFragment.root, True) + + # create element if it doesn't exist + if len(item.prop.getNodes(self.columnHeadings[n], False)) == 0: + item.getXml().createNode(self.columnHeadings[n], "") + else: + # remove any xml subtrees the same name as the nodes in the fragment if first occurence + if self.columnHeadings[:n].count(self.columnHeadings[n]) == 0 and self.existingMetadataMode != self.APPENDMETA and 'APPENDMETA' not in commandOptions: + for child in xmlFragment.root.childNodes: + item.prop.removeNode(self.columnHeadings[n] + "/" + child.nodeName) + + # append xml fragment + parentNode = item.prop.getNodes(self.columnHeadings[n], False)[0] + for child in xmlFragment.root.childNodes: + parentNode.appendChild(child.cloneNode(True)) + else: + # empty cell so clear the text of the node + if self.existingMetadataMode != self.APPENDMETA and 'APPENDMETA' not in commandOptions: + if len(item.prop.getNodes(self.columnHeadings[n], False)) != 0: + parentNode = item.prop.getNodes(self.columnHeadings[n], False)[0] + for child in parentNode.childNodes: + if child.nodeType == Node.TEXT_NODE: + child.nodeValue = "" + + # simple metadata, not an XML fragment + else: + + # split for multi-value field + newValues = meta[n].split(actualDelimiter) + + # remove existing elements if first occurence + if self.columnHeadings[:n].count(self.columnHeadings[n]) == 0 and self.existingMetadataMode != self.APPENDMETA and 'APPENDMETA' not in commandOptions: + item.getXml().removeNode(self.columnHeadings[n]) + + # iterate through new values + for i in range(len(newValues)): + + # display to log if necessary + if self.currentColumns[n][self.COLUMN_DISPLAY] == "YES": + self.echo(" %s: %s" % (self.columnHeadings[n], newValues[i].strip())) + + # create element if not empty string + if newValues[i].strip() != "": + + # if null specified then create empty element + if newValues[i].strip().lower() == "": + newValues[i] = "" + + item.getXml().createNode(self.columnHeadings[n], unicode(newValues[i].strip())) + + # column is set to IGNORE + if self.currentColumns[n][self.COLUMN_DATATYPE] == self.IGNORE: + + # display to log if necessary + if self.currentColumns[n][self.COLUMN_DISPLAY] == "YES": + self.echo(" %s: %s" % (self.columnHeadings[n], meta[n].strip())) + + + # set selected thumbnail to none if applicable + if selectedThumbnail != "": + if selectedThumbnail.upper() == "NONE": + item.getXml().setNode("item/thumbnail", "none") + + + # if necesary set owner and collaborators for new items and new versions + if createNewVersion or createNewItem: + if ownerColumn != -1 and ownerUsername != "": + + # add owner to new item/version + item.getXml().setNode("item/owner", ownerID) + + if collaboratorsColumn != -1: + + # NOTE: EQUELLA automatically clears out collaborators when + # creating new versions so this is actually unneccesary. Not + # actually possible to "append" collaborators to a new version. + if self.existingMetadataMode != self.APPENDMETA and 'APPENDMETA' not in commandOptions: + item.getXml().removeNode("item/collaborativeowners/collaborator") + + # add collaborators to new item/version + for collaboratorID in collaboratorIDs: + item.getXml().createNode("item/collaborativeowners/collaborator", collaboratorID) + + # ############################## + # submit item or cancel editing + # ############################## + n = -1 + wx.GetApp().Yield() + if not self.StopProcessing: + + savedItemID = item.getUUID() + savedItemVersion = item.getVersion() + + + # run Row Post-Script + if self.postScript.strip() != "": + try: + exec self.postScript in { + "IMPORT":0, + "EXPORT":1, + "NEWITEM":0, + "NEWVERSION":1, + "EDITITEM":2, + "DELETEITEM":3, + "mode":0, + "action":scriptAction, + "vars":self.scriptVariables, + "rowData":meta, + "rowCounter":rowCounter, + "testOnly": testOnly, + "institutionUrl":tle.institutionUrl, + "collection":self.collection, + "csvFilePath":self.csvFilePath, + "username":self.username, + "logger":self.logger, + "columnHeadings":self.columnHeadings, + "columnSettings":self.currentColumns, + "successCount":self.successCount, + "errorCount":self.errorCount, + "itemId":savedItemID, + "itemVersion":savedItemVersion, + "xml":item.prop, + "xmldom":item.newDom, + "process":self.process, + "basepath":self.absoluteAttachmentsBasepath, + "sourceIdentifierIndex":sourceIdentifierColumn, + "targetIdentifierIndex":targetIdentifierColumn, + "targetVersionIndex":targetVersionColumn, + "imsmanifest":imsmanifest, + "csvData":self.csvArray, + "ebi":self.ebiScriptObject, + "equella":tle, + } + except: + if self.debug: + raise + else: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + formattedException = "".join(traceback.format_exception_only(exceptionType, exceptionValue))[:-1] + scriptErrorMsg = "An error occured in the Row Post-Script:\n%s (line %s)" % (formattedException, traceback.extract_tb(exceptionTraceback)[-1][1]) + raise Exception, scriptErrorMsg + + + if not self.StopProcessing and not self.Skip: + + if testOnly: + # check to see if test XML files should be produced + if self.saveTestXML: + + # add CAL holding relations metadata if necessary + if calHoldingMetadataTarget != "": + self.addCALRelations(calHoldingMetadataTarget, item.getXml()) + + # create folder if one doesn't exist + xmlFolderName = os.path.join(self.testItemfolder, self.sessionName) + if not os.path.exists(xmlFolderName): + os.makedirs(xmlFolderName) + + # form filename + xmlFilename = os.path.join(xmlFolderName, "ebi-%06d.xml" % rowCounter) + + # save file + fp = file(xmlFilename, 'w') + fp.write(item.newDom.toprettyxml(" ", "\n", self.encoding)) + fp.close() + + # cancel edit (test only) + item.parClient._cancelEdit(item.getUUID(), item.getVersion()) + self.echo(" Item valid for import") + else: + + # add CAL holding relations metadata (if necessary) prior to saving if editing existing item + if calHoldingMetadataTarget != "" and not (createNewItem or createNewVersion): + self.addCALRelations(calHoldingMetadataTarget, item.getXml()) + + # determine bSubmit parameter for saveItem() (controls status of item) + bSubmit = 0 + statusMessage = " in draft status" + if (createNewItem or createNewVersion) and (not self.saveAsDraft and "DRAFT" not in commandOptions): + bSubmit = 1 + statusMessage = "" + + # submit the item + item.submit(bSubmit) + + self.tryPausing("[Paused]") + + # update owner if one specified and existing item being edited + if ownerColumn != -1 and not createNewVersion and not createNewItem and ownerID != "" and ownerUsername != "": + + self.echo(" Setting owner to '%s'" % ownerUsername) + + # only update owner if different from current + if item.getXml().getNode("item/owner") != ownerID: + try: + tle.setOwner(savedItemID, savedItemVersion, ownerID) + except: + exactError = str(sys.exc_info()[1]) + errorDebug = "" + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + errorDebug = "\n" + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + + exactError = self.translateError(exactError) + + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "ERROR: %s%s" % (exactError, errorDebug), style=2) + + # add collaborators if any specified and existing item being edited + if collaboratorsColumn != -1 and not createNewVersion and not createNewItem and len(collaboratorIDs) > 0: + self.echo(" Setting collaborators") + try: + # if remove all existing collaborators unless appending metadata + if self.existingMetadataMode != self.APPENDMETA and 'APPENDMETA' not in commandOptions: + existingCollaboratorIDs = item.getXml().getNodes("item/collaborativeowners/collaborator") + + for existingCollaboratorID in existingCollaboratorIDs: + if existingCollaboratorID not in collaboratorIDs: + tle.removeSharedOwner(savedItemID, savedItemVersion, existingCollaboratorID) + + + # add specified collaborators + for collaboratorID in collaboratorIDs: + if collaboratorID not in existingCollaboratorIDs: + tle.addSharedOwner(savedItemID, savedItemVersion, collaboratorID) + + except: + exactError = str(sys.exc_info()[1]) + errorDebug = "" + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + errorDebug = "\n" + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + + exactError = self.translateError(exactError) + + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "ERROR: %s%s" % (exactError, errorDebug), style=2) + + if createNewItem: + self.echo(" Item successfully imported%s (%s/%s)" % (statusMessage, savedItemID, savedItemVersion)) + else: + if createNewVersion: + self.echo(" New version successfully created" + statusMessage) + else: + self.echo(" Item successfully updated") + + # re-edit and add CAL holding relations metadata if necessary + if (createNewItem or createNewVersion) and calHoldingMetadataTarget != "": + self.echo(" Re-editing to add CAL metadata...") + tle._forceUnlock(savedItemID, savedItemVersion) + item = tle.editItem(savedItemID, savedItemVersion, 'true') + self.addCALRelations(calHoldingMetadataTarget, item.getXml()) + item.submit(bSubmit) + self.echo(" Item successfully updated") + + # increment success count + self.successCount += 1 + else: + item.parClient._cancelEdit(item.getUUID(), item.getVersion()) + if self.Skip: + self.echo(" Row skipped") + + retriesDone = True + + # return Item ID, Item Version and Source Identifier + sourceIdentifier = "" + if sourceIdentifierColumn != -1: + sourceIdentifier = meta[sourceIdentifierColumn].strip() + return savedItemID, savedItemVersion, sourceIdentifier, meta, "" + + except: + exactError = str(sys.exc_info()[1]) + + # check if it is worthwhile recycling the session and retrying + if failCount < self.maxRetry and ("(10054)" in exactError or "(10060)" in exactError or "(10061)" in exactError or "(104)" in exactError): + failCount += 1 + self.echo(" %s. Retrying..." % (exactError)) + + # pause for increasing periods with each fail. 5 seconds, 10 seconds, 15 seconds... so on until maximum number of retries + time.sleep(5 * failCount) + try: + item.parClient._cancelEdit(item.getUUID(), item.getVersion()) + except: + pass + self.tle = None + self.tle = TLEClient(self.institutionUrl, self.username, self.password, self.proxy, self.proxyUsername, self.proxyPassword, self.debug) + + else: + self.errorCount += 1 + + # form error string for debugging + errorDebug = "" + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + errorDebug = "\n" + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + + exactError = self.translateError(exactError) + + if "Collection not found" in exactError: + allRowsError = True + + # add further information if error is with a particular column etc + actionError = "" + if n != -1 and not attemptingUpload: + actionError = " parsing column %s '%s'" % (n + 1, self.columnHeadings[n]) + if attemptingUpload: + actionError = " uploading file" + attemptingUpload = False + + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "ERROR%s: %s%s" % (actionError, exactError, errorDebug), style=2) + + # halt processing if the error will apply to all rows + if allRowsError: + raise Exception, "Halting process" + + # return Item ID, Item Version and Source Identifier + sourceIdentifier = "" + if sourceIdentifierColumn != -1: + sourceIdentifier = meta[sourceIdentifierColumn].strip() + + return savedItemID, savedItemVersion, sourceIdentifier, meta, "ERROR%s: %s%s" % (actionError, exactError, errorDebug) + + def exportCSV(self, + owner, + tle, + itemdefuuid, + collectionIDs, + testOnly, + scheduledRows, + sourceIdentifierColumn, + targetIdentifierColumn, + targetVersionColumn, + commandOptionsColumn, + attachmentLocationsColumn, + collectionColumn, + rowsToBeProcessedCount): + + if not testOnly: + if owner.txtCSVPath.GetValue() != "" and not os.path.isdir(self.csvFilePath): + try: + # test opening file for writing (append test only) + + f = open(self.csvFilePath, "ab") + f.close() + except: + raise Exception, "CSV cannot be written to and may be in use: %s" % self.csvFilePath + + allRowsError = False + self.successCount = 0 + self.errorCount = 0 + processedCounter = 0 + + # run Start Script + if self.startScript.strip() != "": + try: + exec self.startScript in { + "IMPORT":0, + "EXPORT":1, + "mode":1, + "vars":self.scriptVariables, + "testOnly": testOnly, + "institutionUrl":tle.institutionUrl, + "collection":self.collection, + "csvFilePath":self.csvFilePath, + "username":self.username, + "logger":self.logger, + "columnHeadings":self.columnHeadings, + "columnSettings":self.currentColumns, + "successCount":self.successCount, + "errorCount":self.errorCount, + "process":self.process, + "basepath":self.absoluteAttachmentsBasepath, + "sourceIdentifierIndex":sourceIdentifierColumn, + "targetIdentifierIndex":targetIdentifierColumn, + "targetVersionIndex":targetVersionColumn, + "csvData":self.csvArray, + "ebi":self.ebiScriptObject, + "equella":tle, + } + except: + if self.debug: + raise + else: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + formattedException = "".join(traceback.format_exception_only(exceptionType, exceptionValue))[:-1] + scriptErrorMsg = "An error occured in the Start Script:\n%s (line %s)" % (formattedException, traceback.extract_tb(exceptionTraceback)[-1][1]) + raise Exception, scriptErrorMsg + + # check column headings + self.validateColumnHeadings() + + actionString = "" + if self.includeNonLive: + actionString = "Options -> Export non-live items" + if actionString != "": + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + actionString) + + if sourceIdentifierColumn == -1 and targetIdentifierColumn == -1: + + # WHERE Clause + if self.whereClause.strip() != "": + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "WHERE Clause: %s" % self.whereClause) + + # determine which collection to search (seach all collections if Collections column present) + if collectionColumn == -1: + collectionsToSearch = [itemdefuuid] + else: + collectionsToSearch = [] + + # get available + try: + searchResults = tle.search(query='', + itemdefs=collectionsToSearch, + where=self.whereClause.strip(), + onlyLive=not self.includeNonLive, + orderType=0, + reverseOrder=False, + offset=0, + limit=1) + + + except: + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + errorString = "\n" + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + "\n" + else: + errorString = "Error whist attempting to search: " + str(sys.exc_info()[1]) + raise Exception, errorString + + + rowCounter = 0 + available = int(searchResults.getNode("available")) + pageSize = 50 + + # determine how many rows to be processed + itemsToBeProcessedCount = available + if self.rowFilter.strip() != "": + itemsToBeProcessedCount = 0 + for rc in scheduledRows: + if rc <= available: + itemsToBeProcessedCount += 1 + + # echo rows to be processed + scheduledRowsLabel = " " + if itemsToBeProcessedCount > 0: + scheduledRowsLabel = ", all to be exported " + if self.rowFilter != "": + scheduledRowsLabel = ", %s to be processed [%s] " % (itemsToBeProcessedCount, self.rowFilter) + if testOnly: + actionString = "%s item(s) found%s(test only)" % (available, scheduledRowsLabel) + else: + actionString = "%s item(s) found%s" % (available, scheduledRowsLabel) + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + actionString) + + self.owner.progressGauge.SetRange(itemsToBeProcessedCount) + self.owner.progressGauge.SetValue(processedCounter) + self.owner.progressGauge.Show() + + # crop list down to first row + self.csvArray = self.csvArray[:1] + + # outer loop of "pages" of pageSize + pagesRequired = available / pageSize + 1 + offset = 0 + lastScheduledItem = -1 + if len(scheduledRows) > 0: + lastScheduledItem = max(scheduledRows) + + for pageCounter in range(1, pagesRequired + 1): + + searchResults = tle.search(query='', + itemdefs=collectionsToSearch, + where=self.whereClause.strip(), + onlyLive=not self.includeNonLive, + orderType=0, + reverseOrder=False, + offset=offset, + limit=pageSize) + + wx.GetApp().Yield() + + for result in searchResults.iterate("result"): + if not self.StopProcessing: + try: + + # increment rowCounter + rowCounter += 1 + + self.Skip = False + + if self.rowFilter.strip() == "" or rowCounter in scheduledRows: + + processedCounter += 1 + + self.echo("---") + + itemXml = result.getSubtree("xml") + itemID = itemXml.getNode("item/@id") + itemVersion = itemXml.getNode("item/@version") + + # update UI and log + owner.mainStatusBar.SetStatusText("Exporting item %s [%s of %s]" % (rowCounter, processedCounter, itemsToBeProcessedCount), 0) + self.owner.progressGauge.SetValue(processedCounter) + if testOnly: + action = "Exporting item %s/%s (test only)..." % (itemID, itemVersion) + else: + action = "Exporting item %s/%s..." % (itemID, itemVersion) + + + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + " Item %s [%s of %s]: %s" % (rowCounter, processedCounter, itemsToBeProcessedCount, action)) + wx.GetApp().Yield() + + rowData = self.exportItem(rowCounter, + itemXml, + tle, + itemdefuuid, + testOnly, + sourceIdentifierColumn, + targetIdentifierColumn, + targetVersionColumn, + commandOptionsColumn, + attachmentLocationsColumn, + collectionIDs, + self.csvArray) + + if not self.Skip: + if len(self.csvArray) > rowCounter: + self.csvArray[rowCounter] = rowData + else: + self.csvArray.append(rowData) + self.successCount += 1 + + if self.rowFilter.strip() != "" and rowCounter == lastScheduledItem: + break + offset = rowCounter + + except: + exactError = str(sys.exc_info()[1]) + self.errorCount += 1 + + # form error string for debugging + errorDebug = "" + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + errorDebug = "\n" + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + + exactError = self.translateError(exactError) + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "ERROR: %s%s" % (exactError, errorDebug), style=2) + + # halt processing if the error will apply to all rows + if allRowsError: + raise Exception, "Halting process" + + # stop processing + else: + break + if self.StopProcessing: + self.echo("---") + if self.processingStoppedByScript: + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "Export halted") + else: + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "Export halted by user") + break + elif self.rowFilter.strip() != "" and rowCounter == lastScheduledItem: + break + else: + + # echo rows to be processed + scheduledRowsLabel = "all to be exported" + if self.rowFilter != "": + scheduledRowsLabel = "%s to be processed [%s]" % (rowsToBeProcessedCount, self.rowFilter) + if testOnly: + actionString = str(len(self.csvArray) - 1) + " row(s) found, %s (test only)" % (scheduledRowsLabel) + else: + actionString = str(len(self.csvArray) - 1) + " row(s) found, %s" % (scheduledRowsLabel) + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + actionString) + + self.owner.progressGauge.SetRange(rowsToBeProcessedCount) + self.owner.progressGauge.SetValue(processedCounter) + self.owner.progressGauge.Show() + + # iterate through the rows of metadata from the CSV file exporting an item from EQUELLA for each + for rowCounter in scheduledRows: + + if not self.StopProcessing and rowCounter < len(self.csvArray): + try: + processedCounter += 1 + self.Skip = False + + self.echo("---") + + # update UI and log + owner.mainStatusBar.SetStatusText("Exporting row %s [%s of %s]" % (rowCounter, processedCounter, rowsToBeProcessedCount), 0) + self.owner.progressGauge.SetValue(processedCounter) + if testOnly: + action = "Exporting item (test only)..." + else: + action = "Exporting item..." + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + " Row %s [%s of %s]: %s" % (rowCounter, processedCounter, rowsToBeProcessedCount, action)) + wx.GetApp().Yield() + + rowitemdefuuid = itemdefuuid + # override the collection ID if one has been specified in the row + if collectionColumn != -1: + collectionName = self.csvArray[rowCounter][collectionColumn].strip() + if collectionName != "": + if collectionName in collectionIDs: + rowitemdefuuid = collectionIDs[collectionName] + self.echo(" Source collection: '%s'" % collectionName) + else: + raise Exception, "'" + collectionName + "' collection not found" + + + itemFound = False + + # get targeted item version if target version specified + itemVersion = 0 + if targetVersionColumn != -1 and self.csvArray[rowCounter][targetVersionColumn].strip() != "": + try: + itemVersion = int(self.csvArray[rowCounter][targetVersionColumn].strip()) + if itemVersion < -1: + raise Exception, "Invalid item version specified" + except: + raise Exception, "Invalid item version specified" + + # if Source Identifier column specified check if item exists by sourceIdentifier + if sourceIdentifierColumn != -1: + if self.csvArray[rowCounter][sourceIdentifierColumn].strip() != "": + if targetVersionColumn == -1 or self.csvArray[rowCounter][targetVersionColumn].strip() == "": + noVersionSpecified = True + else: + noVersionSpecified = False + + # determine if items versions of any status need to be returned + if itemVersion != 0 or (self.includeNonLive and noVersionSpecified): + onlyLive = False + limit = 50 + else: + onlyLive = True + limit = 1 + + sourceIdentifier = self.csvArray[rowCounter][sourceIdentifierColumn].strip() + self.echo(" Source identifier = " + sourceIdentifier) + if targetVersionColumn != -1 and self.csvArray[rowCounter][targetVersionColumn].strip() != "": + self.echo(" Target version = " + self.csvArray[rowCounter][targetVersionColumn].strip()) + searchFilter = "/xml/" + self.columnHeadings[sourceIdentifierColumn] + "='" + sourceIdentifier + "'" + results = tle.search(0, limit, '', [rowitemdefuuid], searchFilter, query='', onlyLive=onlyLive) + + # if any matches get first matching item for editing + if int(results.getNode("available")) > 0: + if itemVersion == 0 and not (self.includeNonLive and noVersionSpecified): + # get first live version + itemXml = results.getSubtree("result/xml") + itemID = results.getNode("result/xml/item/@id") + itemVersion = results.getNode("result/xml/item/@version") + itemFound = True + else: + if itemVersion > 0: + # find item by item version + for itemResult in results.iterate("result"): + if itemResult.getNode("xml/item/@version") == str(itemVersion): + itemXml = itemResult.getSubtree("xml") + itemID = itemResult.getNode("xml/item/@id") + itemFound = True + break + if not itemFound: + self.echo(" Item not found in EQUELLA") + else: + # find item with highest version + highestVersionFound = 0 + highestLiveVersionFound = 0 + for itemResult in results.iterate("result"): + if int(itemResult.getNode("xml/item/@version")) > highestVersionFound: + + itemXml = itemResult.getSubtree("xml") + itemID = itemResult.getNode("xml/item/@id") + + highestVersionFound = int(itemResult.getNode("xml/item/@version")) + if itemResult.getNode("xml/item/@status") == "live": + highestLiveVersionFound = highestVersionFound + + if itemVersion == 0: + itemVersion = highestLiveVersionFound + else: + itemVersion = highestVersionFound + itemFound = True + if itemFound: + self.echo(" Item exists in EQUELLA (" + itemID + "/" + str(itemVersion) + ")") + else: + self.echo(" Item not found in EQUELLA") + else: + self.echo(" No source identifier specified") + + # if Target Identifier column specified edit item by ID (using latest version of item) + elif targetIdentifierColumn != -1: + if self.csvArray[rowCounter][targetIdentifierColumn].strip() != "": + targetIdentifier = self.csvArray[rowCounter][targetIdentifierColumn].strip() + self.echo(" Target identifier = " + targetIdentifier) + if targetVersionColumn != -1 and self.csvArray[rowCounter][targetVersionColumn].strip() != "": + self.echo(" Target version = " + self.csvArray[rowCounter][targetVersionColumn].strip()) + elif self.includeNonLive: + itemVersion = -1 + + itemID = targetIdentifier + + # try retreiving item + try: + itemXml = tle.getItem(itemID, itemVersion) + itemFound = True + self.echo(" Item exists in EQUELLA (" + itemID + "/" + itemXml.getNode("item/@version") + ")") + except: + self.echo(" Could not find item (" + str(sys.exc_info()[1]) + ")") + else: + self.echo(" No target identifier specified") + + if itemFound: + rowData = self.exportItem(rowCounter, + itemXml, + tle, + itemdefuuid, + testOnly, + sourceIdentifierColumn, + targetIdentifierColumn, + targetVersionColumn, + commandOptionsColumn, + attachmentLocationsColumn, + collectionIDs, + self.csvArray[rowCounter]) + + if not self.Skip: + self.csvArray[rowCounter] = rowData + self.successCount += 1 + + except: + exactError = str(sys.exc_info()[1]) + self.errorCount += 1 + + # form error string for debugging + errorDebug = "" + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + errorDebug = "\n" + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + + exactError = self.translateError(exactError) + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "ERROR: %s%s" % (exactError, errorDebug), style=2) + + # halt processing if the error will apply to all rows + if allRowsError: + raise Exception, "Halting process" + + # stop processing + else: + if self.StopProcessing: + self.echo("---") + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + "Export halted by user") + break + + self.echo("---") + + # run End Script + if self.endScript.strip() != "": + try: + exec self.endScript in { + "IMPORT":0, + "EXPORT":1, + "mode":1, + "vars":self.scriptVariables, + "testOnly": testOnly, + "institutionUrl":tle.institutionUrl, + "collection":self.collection, + "csvFilePath":self.csvFilePath, + "username":self.username, + "logger":self.logger, + "columnHeadings":self.columnHeadings, + "columnSettings":self.currentColumns, + "successCount":self.successCount, + "errorCount":self.errorCount, + "process":self.process, + "basepath":self.absoluteAttachmentsBasepath, + "csvData":self.csvArray, + "sourceIdentifierIndex":sourceIdentifierColumn, + "targetIdentifierIndex":targetIdentifierColumn, + "targetVersionIndex":targetVersionColumn, + "ebi":self.ebiScriptObject, + "equella":tle, + } + except: + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + scriptErrorMsg = "An error occured in the End Script:\n" + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + scriptErrorMsg) + else: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + formattedException = "".join(traceback.format_exception_only(exceptionType, exceptionValue))[:-1] + scriptErrorMsg = "An error occured in the End Script:\n%s (line %s)" % (formattedException, traceback.extract_tb(exceptionTraceback)[-1][1]) + self.echo(time.strftime("%H:%M:%S: ", time.localtime(time.time())) + scriptErrorMsg) + + + # open csv writer and output write local copy to csv + if not testOnly and not os.path.isdir(self.csvFilePath): + f = open(self.csvFilePath, "wb") + writer = UnicodeWriter(f, self.encoding) + writer.writerows(self.csvArray) + f.close() + + def exportItem(self, + rowCounter, + itemXml, + tle, + itemdefuuid, + testOnly, + sourceIdentifierColumn, + targetIdentifierColumn, + targetVersionColumn, + commandOptionsColumn, + attachmentLocationsColumn, + collectionIDs, + oldRowData = None): + + itemID = itemXml.getNode("item/@id") + itemVersion = itemXml.getNode("item/@version") + + self.tryPausing("[Paused]") + + # run Row Pre-Script + if self.preScript.strip() != "": + + ebiScriptObject = EbiScriptObject(self) + + try: + exec self.preScript in { + "IMPORT":0, + "EXPORT":1, + "action":1, + "vars":self.scriptVariables, + "rowCounter":rowCounter, + "testOnly": testOnly, + "institutionUrl":tle.institutionUrl, + "collection":self.collection, + "csvFilePath":self.csvFilePath, + "username":self.username, + "logger":self.logger, + "columnHeadings":self.columnHeadings, + "columnSettings":self.currentColumns, + "successCount":self.successCount, + "errorCount":self.errorCount, + "itemId":itemID, + "itemVersion":itemVersion, + "xml":itemXml, + "xmldom":itemXml.root, + "process":self.process, + "basepath":self.absoluteAttachmentsBasepath, + "sourceIdentifierIndex":sourceIdentifierColumn, + "targetIdentifierIndex":targetIdentifierColumn, + "targetVersionIndex":targetVersionColumn, + "csvData":self.csvArray, + "ebi":self.ebiScriptObject, + "equella":tle, + } + + except: + if self.debug: + raise + else: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + formattedException = "".join(traceback.format_exception_only(exceptionType, exceptionValue))[:-1] + scriptErrorMsg = "An error occured in the Row Pre-Script:\n%s (line %s)" % (formattedException, traceback.extract_tb(exceptionTraceback)[-1][1]) + raise Exception, scriptErrorMsg + + self.csvFilePath = ebiScriptObject.csvFilePath + + rowData = ['']*(len(self.columnHeadings)) + filesDownloaded = [] + command = "" + hyperlinkColumnCount = 0 + attachmentColumnCount = 0 + equellaResourceColumnCount = 0 + + if self.Skip: + self.echo(" Skipping item") + return rowData + + for n in range(0, len(self.columnHeadings)): + cellValues = [] + delimiter = self.currentColumns[n][self.COLUMN_DELIMITER].strip() + + # get metadata values if column datatype uses an xpath + values = [] + if self.currentColumns[n][self.COLUMN_DATATYPE] == self.METADATA or \ + (self.currentColumns[n][self.COLUMN_DATATYPE] in [self.ATTACHMENTLOCATIONS, self.URLS, self.EQUELLARESOURCES, self.CUSTOMATTACHMENTS] and \ + self.columnHeadings[n].strip() != "" and self.columnHeadings[n].strip()[0] != "#"): + + # Get all matching values + values = itemXml.getNodes(self.columnHeadings[n]) + + # detemine how many values to "discount" away (-1 means discount all of them) to + # spread repeating values across columns with same xpaths + valuesUsed = 0 + for i in range(0, n): + if self.columnHeadings[i] == self.columnHeadings[n] and self.currentColumns[i][self.COLUMN_DATATYPE] == self.currentColumns[n][self.COLUMN_DATATYPE]: + if self.currentColumns[i][self.COLUMN_DELIMITER].strip() == "" and valuesUsed != -1: + valuesUsed += 1 + else: + valuesUsed = -1 + + if self.currentColumns[n][self.COLUMN_DATATYPE] == self.METADATA: + # check if column is flagged for XML fragments + if self.currentColumns[n][self.COLUMN_XMLFRAGMENT] == "YES": + + # process node as XML fragment + xmlFragNodes = itemXml.getNodes(self.columnHeadings[n], False) + if len(xmlFragNodes) > 0: + xmlFragment = "" + for childNode in xmlFragNodes[0].childNodes: + # only add node if it is not an empty text node + if (childNode.nodeType == Node.TEXT_NODE and childNode.nodeValue.strip() != "") or (childNode.nodeType != Node.TEXT_NODE): + xmlFragment += childNode.toxml() + + cellValues.append(xmlFragment) + + # not an xml fragment + else: + if len(values) > 0 and valuesUsed != -1: + if delimiter != "": + # get all non-discounted values + cellValues = values[valuesUsed:] + else: + if len(values) > valuesUsed: + cellValues.append(values[valuesUsed]) + + elif self.currentColumns[n][self.COLUMN_DATATYPE] == self.URLS: + hyperlinkColumnCount += 1 + + if len(values) > 0 and valuesUsed != -1: + + attachmentNames = [] + + # calculate first and last index of values applicable to this column + if delimiter != "": + lastValueIndex = len(values) + else: + lastValueIndex = valuesUsed + 1 + + # iterate through the attachment UUIDs applicable to this column to calculate + # cell values whilst downloading files as necessary + for attachmentUUID in values[valuesUsed:lastValueIndex]: + for attachment in itemXml.getSubtree("item/attachments").iterate("attachment"): + filename = attachment.getNode("file").replace(" ", "%20") + if attachment.getNode("uuid") == attachmentUUID: + if attachment.getNode("@type") == "remote": + + # get URL from attachment metadata + self.echo(" Hyperlink: " + filename) + + # collect attachment name + attachmentName = attachment.getNode("description") + if attachmentName == filename: + attachmentName = "" + attachmentNames.append(attachmentName) + + # set cell value + cellValues.append(filename) + elif attachment.getNode("@type") == "local": + if self.debug: + self.echo(" Ignoring: " + filename) + + # find corresponding Attachment Names column to populate with attachment names + attachmentNameColumnCount = 0 + for col in range(0, len(self.currentColumns)): + if self.currentColumns[col][self.COLUMN_DATATYPE] == self.HYPERLINKNAMES: + + attachmentNameColumnCount += 1 + + if attachmentNameColumnCount == hyperlinkColumnCount: + # populate with attachment names + attachmentNameColumnDelimiter = self.currentColumns[col][self.COLUMN_DELIMITER].strip() + if attachmentNameColumnDelimiter != "": + rowData[col] = attachmentNameColumnDelimiter.join(attachmentNames) + else: + rowData[col] = attachmentNames[0] + break + + elif self.currentColumns[n][self.COLUMN_DATATYPE] == self.ATTACHMENTLOCATIONS: + attachmentColumnCount += 1 + + if len(values) > 0 and valuesUsed != -1: + + # get absolute path to file relative to base path (from Options) and then relative to csv folder + filesfolder = self.absoluteAttachmentsBasepath + attachmentNames = [] + attachmentNamesZip = [] + zipFiles = [] + + # get item URL (used for downloading files) + itemUrl = self.institutionUrl + "/file/" + itemID + "/" + itemVersion + "/" + + # calculate first and last index of values applicable to this column + if delimiter != "": + lastValueIndex = len(values) + else: + lastValueIndex = valuesUsed + 1 + + # iterate through the attachment UUIDs applicable to this column to calculate + # cell values whilst downloading files as necessary + for attachmentUUID in values[valuesUsed:lastValueIndex]: + for attachment in itemXml.getSubtree("item/attachments").iterate("attachment"): + if attachment.getNode("uuid") == attachmentUUID: + filename = attachment.getNode("file") + if attachment.getNode("@type") == "local": + if filename.find("/") == -1: + # download simple file + filepath = os.path.join(filesfolder, filename) + fileUrl = itemUrl + urllib.quote(filename) + + if not testOnly and not fileUrl in filesDownloaded: + self.echo(" Attachment: " + filename) + + # "deconflict" files of same name + filepath = self.deconflict(filepath, self.exportedFiles, self.overwriteMode) + + tle.getFile(fileUrl, filepath) + filesDownloaded.append(fileUrl) + self.exportedFiles.append(filepath) + + # collect attachment name + attachmentName = attachment.getNode("description") + if attachmentName == filename: + attachmentName = "" + attachmentNames.append(attachmentName) + + # set cell value + if not filename in cellValues: + cellValues.append(os.path.relpath(filepath, filesfolder)) + else: + # possibly a zip file + rootFolder = filename[:filename.find("/")] + if rootFolder.endswith(".zip"): + # download zip file + zipfilename = rootFolder + relfilename = filename[filename.find("/") + 1:] + filepath = os.path.join(filesfolder, zipfilename) + fileUrl = itemUrl + "_zips/" + urllib.quote(zipfilename) + + if not testOnly and not fileUrl in filesDownloaded: + self.echo(" Attachment (ZIP): " + zipfilename) + + # "deconflict" files of same name + filepath = self.deconflict(filepath, self.exportedFiles, self.overwriteMode) + + tle.getFile(fileUrl, filepath) + filesDownloaded.append(fileUrl) + self.exportedFiles.append(filepath) + + # collect attachment name + attachmentNamesZip.append([relfilename, attachment.getNode("description")]) + + # set command + if command == "": + command = "UNZIP" + elif command != "UNZIP": + command = "AUTO" + + # add to cell values if zip not already there + if zipfilename not in zipFiles: + cellValues.append(os.path.relpath(filepath, filesfolder)) + zipFiles.append(zipfilename) + + elif attachment.getNode("@type") == "custom" and attachment.getNode("type") == "scorm": + # download SCORM package + filepath = os.path.join(filesfolder, filename) + + # "deconflict" files of same name + filepath = self.deconflict(filepath, self.exportedFiles, self.overwriteMode) + + fileUrl = itemUrl + "_SCORM/" + urllib.quote(filename) + + if not testOnly and not fileUrl in filesDownloaded: + self.echo(" Attachment (SCORM): " + filename) + tle.getFile(fileUrl, filepath) + filesDownloaded.append(fileUrl) + self.exportedFiles.append(filepath) + + # collect attachment name + attachmentName = attachment.getNode("description") + if attachmentName == filename: + attachmentName = "" + attachmentNames = [] + attachmentNames.append(attachmentName) + + # set command + if command == "": + command = "IMS" + elif command != "IMS": + command = "AUTO" + + # set cell value + if not filename in cellValues: + cellValues.append(filename) + elif attachment.getNode("@type") == "remote": + if self.debug: + self.echo(" Ignoring: " + filename) + else: + # attachment not supported for export + self.echo(" Unknown or unsupported attachment: " + filename) + + if itemXml.getNode("item/itembody/packagefile/@uuid") == attachmentUUID: + filename = itemXml.getNode("item/itembody/packagefile") + + # download IMS package + filepath = os.path.join(filesfolder, filename) + + # "deconflict" files of same name + filepath = self.deconflict(filepath, self.exportedFiles, self.overwriteMode) + + fileUrl = itemUrl + "_IMS/" + urllib.quote(filename) + + if not testOnly and not fileUrl in filesDownloaded: + self.echo(" Attachment (IMS): " + filename) + tle.getFile(fileUrl, filepath) + filesDownloaded.append(fileUrl) + self.exportedFiles.append(filepath) + + # collect attachment name + attachmentName = itemXml.getNode("item/itembody/packagefile/@name") + if attachmentName == filename: + attachmentName = "" + attachmentNames = [] + attachmentNames.append(attachmentName) + + # set command + if command == "": + command = "IMS" + elif command != "IMS": + command = "AUTO" + + # set cell value + if not filename in cellValues: + cellValues.append(filename) + + # find corresponding Attachment Names column to populate with attachment names + attachmentNameColumnCount = 0 + for col in range(0, len(self.currentColumns)): + if self.currentColumns[col][self.COLUMN_DATATYPE] == self.ATTACHMENTNAMES: + + attachmentNameColumnCount += 1 + + if attachmentNameColumnCount == attachmentColumnCount: + # populate with attachment names + if len(attachmentNamesZip) == 0: + attachmentNameColumnDelimiter = self.currentColumns[col][self.COLUMN_DELIMITER].strip() + if attachmentNameColumnDelimiter != "": + rowData[col] = attachmentNameColumnDelimiter.join(attachmentNames) + else: + rowData[col] = attachmentNames[0] + else: + attachmentName = "(" + for pair in attachmentNamesZip: + attachmentName += "(" + attachmentName += "\"" + pair[0] + "\",\"" + pair[1] + "\"" + attachmentName += ")," + attachmentName = attachmentName[:-1] + attachmentName += ")" + rowData[col] = attachmentName + break + + elif self.currentColumns[n][self.COLUMN_DATATYPE] == self.OWNER: + + self.echo(" Exporting owner") + + # get owner ID + userID = itemXml.getNodes("item/owner")[0] + + try: + # get username from user ID + username = tle.getUser(userID).getNode("username") + except: + + # handle inability to retrieve username from user ID + if self.saveNonexistentUsernamesAsIDs: + self.echo(" User ID '%s' not found so exporting raw." % (userID)) + username = userID + else: + raise Exception, "No user found with matching user ID: %s" % userID + + cellValues = [username] + + elif self.currentColumns[n][self.COLUMN_DATATYPE] == self.ITEMID: + cellValues = [itemID] + + elif self.currentColumns[n][self.COLUMN_DATATYPE] == self.ITEMVERSION: + cellValues = [itemVersion] + + elif self.currentColumns[n][self.COLUMN_DATATYPE] == self.COLLABORATORS: + + self.echo(" Exporting collaborators") + + # get collaborators + userIDs = itemXml.getNodes("item/collaborativeowners/collaborator") + + if delimiter == "" and len(userIDs) > 1: + userIDs = userIDs[:1] + + cellValues = [] + + for userID in userIDs: + try: + # get username from user ID + username = tle.getUser(userID).getNode("username") + except: + + # handle inability to retrieve username from user ID + if self.saveNonexistentUsernamesAsIDs: + self.echo(" User ID '%s' not found so exporting raw." % (userID)) + username = userID + else: + raise Exception, "No user found with matching user ID: %s" % userID + + cellValues.append(username) + + elif self.currentColumns[n][self.COLUMN_DATATYPE] == self.COLLECTION: + collID = itemXml.getNode("item/@itemdefid") + collName = (key for key,value in collectionIDs.items() if value==collID).next() + cellValues = [collName] + + elif self.currentColumns[n][self.COLUMN_DATATYPE] == self.ITEMID: + cellValues = [itemID] + + elif self.currentColumns[n][self.COLUMN_DATATYPE] == self.ITEMVERSION: + cellValues = [itemVersion] + + elif self.currentColumns[n][self.COLUMN_DATATYPE] == self.ATTACHMENTLOCATIONS: + attachmentColumnCount += 1 + + elif self.currentColumns[n][self.COLUMN_DATATYPE] in [self.IGNORE, self.TARGETIDENTIFIER, self.TARGETVERSION]: + if oldRowData != None: + cellValues = [oldRowData[n]] + + # delimit cell values + cellValue = delimiter.join(cellValues) + + # display to log if necessary + if self.currentColumns[n][self.COLUMN_DISPLAY] == "YES": + self.echo(" %s: %s" % (self.columnHeadings[n], cellValue)) + + # populate delimited list of cell values in row data + if len(cellValues) > 0: + rowData[n] = cellValue + + # add Commands cell + if commandOptionsColumn != -1: + rowData[commandOptionsColumn] = command + + # run Row Post-Script + if self.postScript.strip() != "": + try: + exec self.postScript in { + "IMPORT":0, + "EXPORT":1, + "action":1, + "vars":self.scriptVariables, + "rowData":rowData, + "rowCounter":rowCounter, + "testOnly": testOnly, + "institutionUrl":tle.institutionUrl, + "collection":self.collection, + "csvFilePath":self.csvFilePath, + "username":self.username, + "logger":self.logger, + "columnHeadings":self.columnHeadings, + "columnSettings":self.currentColumns, + "successCount":self.successCount, + "errorCount":self.errorCount, + "itemId":itemID, + "itemVersion":itemVersion, + "xml":itemXml, + "xmldom":itemXml.root, + "process":self.process, + "basepath":self.absoluteAttachmentsBasepath, + "sourceIdentifierIndex":sourceIdentifierColumn, + "targetIdentifierIndex":targetIdentifierColumn, + "targetVersionIndex":targetVersionColumn, + "csvData":self.csvArray, + "ebi":self.ebiScriptObject, + "equella":tle, + } + except: + if self.debug: + raise + else: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + formattedException = "".join(traceback.format_exception_only(exceptionType, exceptionValue))[:-1] + scriptErrorMsg = "An error occured in the Row Post-Script:\n%s (line %s)" % (formattedException, traceback.extract_tb(exceptionTraceback)[-1][1]) + raise Exception, scriptErrorMsg + + + if testOnly: + self.echo(" Item valid for export") + else: + self.echo(" Item successfully exported") + return rowData + + def deconflict(self, filepath, exportedFiles, overwriteMode): + deconflictedFilepath = filepath + + if overwriteMode == self.OVERWRITEEXISTING: + if filepath in exportedFiles: + conflictFolderNumber = 0 + conflictFolderUsed = True + while conflictFolderUsed: + conflictFolderNumber += 1 + conflictFilepath = os.path.join(os.path.dirname(filepath), str(conflictFolderNumber), os.path.basename(filepath)) + if conflictFilepath not in exportedFiles: + conflictFolderUsed = False + if not os.path.exists(os.path.dirname(conflictFilepath)): + os.makedirs(os.path.dirname(conflictFilepath)) + deconflictedFilepath = conflictFilepath + + elif overwriteMode == self.OVERWRITENONE: + if os.path.isfile(filepath): + conflictFolderNumber = 0 + conflictFolderUsed = True + while conflictFolderUsed: + conflictFolderNumber += 1 + conflictFilepath = os.path.join(os.path.dirname(filepath), str(conflictFolderNumber), os.path.basename(filepath)) + if not os.path.isfile(conflictFilepath): + conflictFolderUsed = False + if not os.path.exists(os.path.dirname(conflictFilepath)): + os.makedirs(os.path.dirname(conflictFilepath)) + deconflictedFilepath = conflictFilepath + + return deconflictedFilepath + +class UnicodeWriter: + def __init__(self, f, encoding="utf-8", dialect=csv.excel, **kwds): + # Redirect output to a queue + self.queue = cStringIO.StringIO() + self.writer = csv.writer(self.queue, dialect=dialect, **kwds) + self.stream = f + self.encoder = codecs.getincrementalencoder(encoding)() + + def writerow(self, row): + self.writer.writerow([s.encode("utf-8") for s in row]) + # Fetch UTF-8 output from the queue ... + data = self.queue.getvalue() + data = data.decode("utf-8") + # ... and reencode it into the target encoding + data = self.encoder.encode(data) + # write to the target stream + self.stream.write(data) + # empty queue + self.queue.truncate(0) + + def writerows(self, rows): + for row in rows: + self.writerow(row) + +# script object used by EBI scripts +class EbiScriptObject(object): + def __init__(self, parent): + self.parent = parent + + def getCsvFilePath(self): + return self.parent.csvFilePath + def setCsvFilePath(self, value): + self.parent.csvFilePath = value + csvFilePath = property(getCsvFilePath, setCsvFilePath) + + def getBasepath(self): + return self.parent.absoluteAttachmentsBasepath + def setBasepath(self, value): + self.parent.absoluteAttachmentsBasepath = value + basepath = property(getBasepath, setBasepath) + + def loadCsv(self): + self.parent.loadCSV(self.parent.owner) + +# Logger class only used by EBI scripts +class Logger: + def __init__(self, parent): + self.parent = parent + def log(self, entry, display = True, log = True): + if not isinstance(entry, basestring): + entry = str(entry) + self.parent.echo(entry=entry, display=display, log=log, style=3) + +# Process class only used by EBI scripts +class Process: + def __init__(self, parent): + self.parent = parent + self.halted = False + def halt(self): + self.parent.StopProcessing = True + self.parent.processingStoppedByScript = True + self.halted = True + def skip(self): + self.parent.Skip = True + diff --git a/package-mac/source/ebi/MainFrame.py b/package-mac/source/ebi/MainFrame.py new file mode 100644 index 0000000..1ca2234 --- /dev/null +++ b/package-mac/source/ebi/MainFrame.py @@ -0,0 +1,2564 @@ +#Boa:Frame:MainFrame + +# Author: Jim Kurian, Pearson plc. +# Date: October 2014 +# +# Graphical user interface for the EQUELLA Bulk Importer. Utilizes +# wxPython to render an OS-native UI. Main purpose of the UI is to +# specify CSV settings, EQUELLA settings, save settings to file and start +# and stop import runs. Requires ebi.py to launch it. + +import sys, time, keyword +import os, traceback +import wx +import wx.grid +import wx.stc as stc +import OptionsDialog +import csv, codecs, cStringIO +import random, platform +import Engine +from xml.dom.minidom import Document, parse, parseString +import urllib2 +import ConfigParser +from equellaclient41 import * + +def create(parent): + return MainFrame(parent) + +[wxID_MAINFRAME, + wxID_MAINFRAMECOLUMNSGRID, wxID_MAINFRAMECONNECTIONPANEL, + wxID_MAINFRAMEGRIDPANEL, + wxID_MAINFRAMEMAINSTATUSBAR, wxID_MAINFRAMEMAINTOOLBAR, + wxID_MAINFRAMETXTCSVPATH, + wxID_MAINFRAMETXTPASSWORD, + wxID_MAINFRAMETXTUSERNAME, wxID_MAINFRAMESTATICTEXT7, + wxID_MAINFRAMELOGPANEL, wxID_MAINFRAMECSVPANEL +] = [wx.NewId() for _init_ctrls in range(12)] + +[wxID_MAINFRAMEMAINTOOLBARABOUT, wxID_MAINFRAMEMAINTOOLBAROPEN, + wxID_MAINFRAMEMAINTOOLBARSAVE, wxID_MAINFRAMEMAINTOOLBARSTOP, + wxID_MAINFRAMEMAINTOOLBARPAUSE, wxID_MAINFRAMEMAINTOOLBAROPTIONS, +] = [wx.NewId() for _init_coll_mainToolbar_Tools in range(6)] + + +class ConnectionPage(wx.Panel): + def __init__(self, parent): + wx.Panel.__init__(self, parent) + +class CSVPage(wx.Panel): + def __init__(self, parent): + wx.Panel.__init__(self, parent) + +class OptionsPage(wx.Panel): + def __init__(self, parent): + wx.Panel.__init__(self, parent) + +class LogPage(wx.Panel): + def __init__(self, parent, owner): + self.parent = parent + self.owner = owner + wx.Panel.__init__(self, parent) + +class MainFrame(wx.Frame): + + def _init_coll_mainToolbar_Tools(self, parent): + + # generated method, don't edit + parent.DoAddTool(bitmap=wx.Bitmap(os.path.join(self.scriptFolder, + u'fileopen.png'), wx.BITMAP_TYPE_PNG), bmpDisabled=wx.NullBitmap, + id=wxID_MAINFRAMEMAINTOOLBAROPEN, kind=wx.ITEM_NORMAL, label='Open', + longHelp='', shortHelp=u'Open Settings File') + parent.DoAddTool(bitmap=wx.Bitmap(os.path.join(self.scriptFolder, + u'filesave.png'), wx.BITMAP_TYPE_PNG), bmpDisabled=wx.NullBitmap, + id=wxID_MAINFRAMEMAINTOOLBARSAVE, kind=wx.ITEM_NORMAL, label='Save', + longHelp=u'', shortHelp=u'Save Settings (' + self.ctrlButton + '+S)') + parent.DoAddTool(bitmap=wx.Bitmap(os.path.join(self.scriptFolder, + u'gtk-stop.png'), wx.BITMAP_TYPE_PNG), bmpDisabled=wx.NullBitmap, + id=wxID_MAINFRAMEMAINTOOLBARSTOP, kind=wx.ITEM_NORMAL, label='Stop', + longHelp=u'', shortHelp=u'Stop Processing') + parent.DoAddTool(bitmap=wx.Bitmap(os.path.join(self.scriptFolder, + u'pause.png'), wx.BITMAP_TYPE_PNG), bmpDisabled=wx.NullBitmap, + id=wxID_MAINFRAMEMAINTOOLBARPAUSE, kind=wx.ITEM_NORMAL, label='Pause', + longHelp=u'', shortHelp=u'Pause/Unpause Processing') + parent.DoAddTool(bitmap=wx.Bitmap(os.path.join(self.scriptFolder, + u'options.png'), wx.BITMAP_TYPE_PNG), bmpDisabled=wx.NullBitmap, + id=wxID_MAINFRAMEMAINTOOLBAROPTIONS, kind=wx.ITEM_NORMAL, label='Preferences', + longHelp=u'', shortHelp=u'Preferences') + parent.DoAddTool(bitmap=wx.Bitmap(os.path.join(self.scriptFolder, + u'gtk-help.png'), wx.BITMAP_TYPE_PNG), bmpDisabled=wx.NullBitmap, + id=wxID_MAINFRAMEMAINTOOLBARABOUT, kind=wx.ITEM_NORMAL, label='About', + longHelp=u'', shortHelp=u'About EQUELLA Bulk Importer') + self.Bind(wx.EVT_TOOL, self.OnMainToolbarSaveTool, + id=wxID_MAINFRAMEMAINTOOLBARSAVE) + self.Bind(wx.EVT_TOOL, self.OnMainToolbarSaveTool, + id=wxID_MAINFRAMEMAINTOOLBAROPEN) + self.Bind(wx.EVT_TOOL, self.OnMainToolbarSaveTool, + id=wxID_MAINFRAMEMAINTOOLBARSTOP) + self.Bind(wx.EVT_TOOL, self.OnMainToolbarSaveTool, + id=wxID_MAINFRAMEMAINTOOLBARPAUSE) + self.Bind(wx.EVT_TOOL, self.OnMainToolbarSaveTool, + id=wxID_MAINFRAMEMAINTOOLBAROPTIONS) + self.Bind(wx.EVT_TOOL, self.OnMainToolbarSaveTool, + id=wxID_MAINFRAMEMAINTOOLBARABOUT) + + parent.Realize() + + def _init_coll_mainStatusBar_Fields(self, parent): + # generated method, don't edit + parent.SetFieldsCount(3) + + parent.SetStatusText('Ready', 0) + parent.SetStatusText("", 1) + parent.SetStatusText("", 2) + + parent.SetStatusWidths([-2, -1, -1]) + parent.SetMinHeight(30) + + + def _init_sizers(self): + # generated method, don't edit + self.mainBoxSizer = wx.BoxSizer(orient=wx.VERTICAL) + + self.mainBoxSizer.AddWindow(self.mainToolbar, 0, border=0, flag=wx.EXPAND) + self.mainBoxSizer.AddWindow(self.nb, 1, border=0, flag=wx.EXPAND) + self.mainBoxSizer.AddWindow(self.mainStatusBar, 0, border=0, flag=wx.EXPAND) + self.SetSizer(self.mainBoxSizer) + + def _init_ctrls(self, prnt): + + self.scriptFolder = sys.path[0] + if self.scriptFolder.endswith(".zip"): + self.scriptFolder = os.path.dirname(self.scriptFolder) + + # set ctrl or cmd tooltip depending on platform + self.ctrlButton = "Ctrl" + if platform.system() == "Darwin": + self.ctrlButton = "Cmd" + + # generated method, don't edit + wx.Frame.__init__(self, id=wxID_MAINFRAME, name='', parent=prnt, + pos=wx.Point(617, 243), size=wx.Size(1022, 575), + style=wx.DEFAULT_FRAME_STYLE, title='EQUELLA Bulk Importer') + self.SetClientSize(wx.Size(1012, 575)) + self.SetMinSize(wx.Size(1012, 575)) + self.SetAutoLayout(False) + self.SetThemeEnabled(False) + + # notebook (tabs) + self.nb = wx.Notebook(parent=self, pos=wx.Point(0, 47)) + self.connectionPage = ConnectionPage(self.nb) + self.nb.AddPage(self.connectionPage, "Connection") + self.csvPage = CSVPage(self.nb) + self.nb.AddPage(self.csvPage, "CSV") + self.optionsPage = OptionsPage(self.nb) + self.nb.AddPage(self.optionsPage, "Options") + self.logPage = LogPage(self.nb, self) + self.nb.AddPage(self.logPage, "Log") + + self.columnsGrid = wx.grid.Grid(id=wxID_MAINFRAMECOLUMNSGRID, + name=u'columnsGrid', parent=self.csvPage, pos=wx.Point(0, 125), + size=wx.Size(1006, 281), style=0) + self.columnsGrid.SetToolTipString(u'CSV column settings') + self.columnsGrid.Bind(wx.grid.EVT_GRID_CELL_CHANGE, + self.OnColumnsGridGridCellChange) + + self.log = Log(self.logPage) + + + # Connection Page controls + self.ConnectionSizer = wx.FlexGridSizer(1, 2) + + self.staticBox = wx.StaticBox(self.connectionPage, -1, "", size=wx.Size(600, 300)) + self.borderBox = wx.StaticBoxSizer(self.staticBox, wx.VERTICAL) + self.borderBox.Add(self.ConnectionSizer, 0, wx.ALIGN_CENTRE|wx.ALL, 5) + + self.connectionPage.SetSizer(self.ConnectionSizer) + + self.ConnectionSizer.AddSpacer(25) + self.ConnectionSizer.AddSpacer(25) + + label = wx.StaticText(id=-1, + label=u'Institution URL:', name='staticText1', + parent=self.connectionPage, size=wx.Size(103, + 17), style=wx.ALIGN_RIGHT) + self.ConnectionSizer.Add(label, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 4) + + self.txtInstitutionUrl = wx.TextCtrl(id=-1, + name=u'txtInstitutionUrl', parent=self.connectionPage, + size=wx.Size(422, 21), style=0, value=u'') + self.txtInstitutionUrl.SetHelpText(u'Institution URL (e.g. "http://equella.myinstitution.edu/training")') + self.txtInstitutionUrl.SetToolTipString(u'EQUELLA institution URL (e.g. "http://equella.myinstitution.org/training")') + self.ConnectionSizer.Add(self.txtInstitutionUrl) + + label = wx.StaticText(id=-1, + label=u'Username:', name='staticText1', + parent=self.connectionPage, size=wx.Size(103, + 17), style=wx.ALIGN_RIGHT) + self.ConnectionSizer.Add(label, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 4) + + self.txtUsername = wx.TextCtrl(id=wxID_MAINFRAMETXTUSERNAME, + name=u'txtUsername', parent=self.connectionPage, + pos=wx.Point(104, 27), size=wx.Size(155, 21), style=0, value=u'') + self.txtUsername.SetToolTipString(u'EQUELLA username') + self.ConnectionSizer.Add(self.txtUsername) + + label = wx.StaticText(id=-1, + label=u'Password:', name='staticText1', + parent=self.connectionPage, size=wx.Size(103, + 17), style=wx.ALIGN_RIGHT) + self.ConnectionSizer.Add(label, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 4) + + self.txtPassword = wx.TextCtrl(id=wxID_MAINFRAMETXTPASSWORD, + name=u'txtPassword', parent=self.connectionPage, + pos=wx.Point(104, 52), size=wx.Size(155, 21), + style=wx.TE_PASSWORD, value=u'') + self.txtPassword.Show(True) + self.txtPassword.SetToolTipString(u'EQUELLA password') + self.ConnectionSizer.Add(self.txtPassword) + + self.btnGetCollections = wx.Button(id=-1, + label=u'Test / Get Collections', name=u'btnGetCollections', + parent=self.connectionPage, pos=wx.Point(529, 2), + size=wx.Size(155, 35), style=0) + self.btnGetCollections.SetToolTipString(u'Connect to EQUELLA and retrieve available collections') + self.btnGetCollections.Bind(wx.EVT_BUTTON, self.OnBtnGetCollectionsButton) + self.ConnectionSizer.AddSpacer(15) + self.ConnectionSizer.Add(self.btnGetCollections) + + self.ConnectionSizer.AddSpacer(15) + self.ConnectionSizer.AddSpacer(15) + + label = wx.StaticText(id=-1, + label=u'Collection:', name='staticText4', + parent=self.connectionPage, + size=wx.Size(103, 17), style=wx.ALIGN_RIGHT) + self.ConnectionSizer.Add(label, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 4) + + self.cmbCollections = wx.Choice(choices=[], + id=-1, name=u'cmbCollections', + parent=self.connectionPage, pos=wx.Point(309, 27), + size=wx.Size(422, 21), style=0) + self.cmbCollections.SetToolTipString(u'EQUELLA collection to import CSV content into') + self.ConnectionSizer.Add(self.cmbCollections) + + # CSV Page controls + self.CsvSizer = wx.BoxSizer(orient=wx.VERTICAL) + self.csvPage.SetSizer(self.CsvSizer) + + box = wx.BoxSizer(wx.HORIZONTAL) + self.CsvSizer.Add(box) + + label = wx.StaticText(self.csvPage, -1, "CSV:") + box.Add(label, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 2) + self.txtCSVPath = wx.TextCtrl(id=wxID_MAINFRAMETXTCSVPATH, + name=u'txtCSVPath', parent=self.csvPage, size=wx.Size(445, 21), style=0, value=u'') + self.txtCSVPath.SetToolTipString(u'File path to CSV') + box.Add(self.txtCSVPath, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 2) + + self.btnBrowseCSV = wx.Button(id=-1, + label=u'Browse...', name=u'btnBrowseCSV', + parent=self.csvPage, + size=wx.Size(90, 23), style=0) + self.btnBrowseCSV.SetHelpText(u'Browse the computer for a CSV file') + self.btnBrowseCSV.SetToolTipString(u'Browse the computer for a CSV to load') + self.btnBrowseCSV.Bind(wx.EVT_BUTTON, self.OnBtnBrowseCSVButton) + box.Add(self.btnBrowseCSV, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 2) + + self.btnReloadCSV = wx.Button(id=-1, + label=u'Reload CSV', name=u'btnReloadCSV', + parent=self.csvPage, pos=wx.Point(846, 2), + size=wx.Size(90, 23), style=0) + self.btnReloadCSV.SetHelpText(u'Reload the CSV to update the settings for any column heading changes in the CSV') + self.btnReloadCSV.SetToolTipString(u'Reload the columns and update for any column heading changes in the CSV') + self.btnReloadCSV.Bind(wx.EVT_BUTTON, self.OnBtnReloadCSVButton, id=-1) + box.Add(self.btnReloadCSV, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 2) + + box = wx.BoxSizer(wx.HORIZONTAL) + self.CsvSizer.Add(box) + + label = wx.StaticText(self.csvPage, -1, "Encoding:") + box.Add(label, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 2) + self.cmbEncoding = wx.Choice(parent=self.csvPage, choices=[], id=-1, name=u'cmbEncoding', size=wx.Size(75, 21), style=0) + self.cmbEncoding.SetToolTipString(u'Encoding used to read CSV file') + box.Add(self.cmbEncoding, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 2) + + for encoding in self.encodingOptions: + self.cmbEncoding.Append(encoding) + self.cmbEncoding.SetSelection(0) + self.CsvSizer.Add(self.columnsGrid, 1, wx.EXPAND) + + label = wx.StaticText(id=-1, + label=u'Row filter:', name='staticText6', + parent=self.csvPage, pos=wx.Point(753, 31), + size=wx.Size(77, 13), style=wx.ALIGN_RIGHT) + box.Add(label, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 2) + self.txtRowFilter = wx.TextCtrl(id=-1, + name=u'txtRowFilter', parent=self.csvPage, + pos=wx.Point(830, 27), size=wx.Size(158, 21), style=0, value=u'') + self.txtRowFilter.SetHelpText(u'Restrict rows to be processed (e.g. "1,3,5-9,4")') + self.txtRowFilter.SetToolTipString(u'Restrict rows in the CSV to be processed (e.g. "2,3,7-12,4,1")') + box.Add(self.txtRowFilter, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 2) + + # options page + labelWidth = 70 + indentWidth = 20 + padding = 4 + + self.optionsSizer = wx.BoxSizer(orient=wx.VERTICAL) + self.optionsPage.SetSizer(self.optionsSizer) + + box = wx.BoxSizer(wx.HORIZONTAL) + label = wx.StaticText(self.optionsPage, -1, "Use following base path for attachments (path to CSV directory used if left blank):") + box.Add(label, 0, wx.ALIGN_CENTER_VERTICAL|wx.LEFT|wx.TOP, padding + 2) + self.optionsSizer.Add(box, 0, wx.GROW|wx.ALIGN_CENTER_VERTICAL|wx.ALL, 1) + + box = wx.BoxSizer(wx.HORIZONTAL) + self.txtAttachmentsBasepath = wx.TextCtrl(self.optionsPage, -1, "", size=wx.Size(400,-1)) + box.Add(self.txtAttachmentsBasepath, 1, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 1) + btnBrowseBasePath = wx.Button(id=-1, + label=u'Browse...', name=u'btnBrowseBasePath', + parent=self.optionsPage, size=wx.Size(88, 23), style=0) + box.Add(btnBrowseBasePath, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 1) + self.Bind(wx.EVT_BUTTON, self.OnBtnBrowseBasePath, btnBrowseBasePath) + self.optionsSizer.Add(box, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 1) + + self.chkSaveAsDraft = wx.CheckBox(parent=self.optionsPage, id=-1, + label=u'Create new items and item versions in draft status (override DRAFT command)', name=u'chkSaveAsDraft', style=0) + self.chkSaveAsDraft.SetValue(False) + self.chkSaveAsDraft.SetToolTipString(u'Create new items and new item versions in draft status (overrides DRAFT command option)') + self.optionsSizer.Add(self.chkSaveAsDraft, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + + self.chkSaveTestXml = wx.CheckBox(parent=self.optionsPage, id=-1, + label=u'Save test XML', name=u'chkSaveTestXml', style=0) + self.chkSaveTestXml.SetValue(False) + self.chkSaveTestXml.SetToolTipString(u'Output example XML files of item metadata from test imports') + self.optionsSizer.Add(self.chkSaveTestXml, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + + line = wx.StaticLine(self.optionsPage, -1, size=(20,-1), style=wx.LI_HORIZONTAL) + self.optionsSizer.Add(line, 0, wx.GROW|wx.ALIGN_CENTER_VERTICAL|wx.RIGHT|wx.TOP, padding) + + box = wx.BoxSizer(wx.HORIZONTAL) + label = wx.StaticText(self.optionsPage, -1, "When updating existing items:", size=(300, -1)) + box.Add(label, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + self.optionsSizer.Add(box, 0, wx.GROW|wx.ALIGN_CENTER_VERTICAL|wx.TOP, padding) + + box = wx.BoxSizer(wx.HORIZONTAL) + self.optionsSizer.Add(box, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + + self.cmbExistingMetadataMode = wx.Choice(choices=[], + id=-1, name=u'cmbExistingMetadataMode', + parent=self.optionsPage, size=wx.Size(210, 21), style=0) + self.cmbExistingMetadataMode.Append("Clear all existing metadata") + self.cmbExistingMetadataMode.Append("Replace only specified metadata") + self.cmbExistingMetadataMode.Append("Append to existing metadata") + self.cmbExistingMetadataMode.SetSelection(0) + label = wx.StaticText(self.optionsPage, -1, "Existing Metadata:", size=(-1, -1), style=wx.ALIGN_RIGHT) + box.AddSpacer(wx.Size(indentWidth, -1), border=0, flag=0) + box.Add(label, 0, wx.ALIGN_TOP|wx.ALL, padding) + box.Add(self.cmbExistingMetadataMode, 1, wx.TOP|wx.ALL, padding) + + self.chkAppendAttachments = wx.CheckBox(parent=self.optionsPage, id=-1, + label=u'Append Attachments', name=u'chkAppendAttachments', style=0) + self.chkAppendAttachments.SetValue(False) + self.chkAppendAttachments.SetToolTipString(u'Leave attachments of existing items untouched and append specified attachments') + box.AddSpacer(wx.Size(indentWidth, -1), border=0, flag=0) + box.Add(self.chkAppendAttachments, 1, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 0) + + self.chkCreateNewVersions = wx.CheckBox(parent=self.optionsPage, id=-1, + label=u'Create new versions', name=u'chkCreateNewVersions', style=0) + self.chkCreateNewVersions.SetValue(False) + self.chkCreateNewVersions.SetToolTipString(u'When updating existing items create new versions (overrides VERSION command option)') + box.Add(self.chkCreateNewVersions, 1, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + + line = wx.StaticLine(self.optionsPage, -1, size=(20,-1), style=wx.LI_HORIZONTAL) + self.optionsSizer.Add(line, 0, wx.GROW|wx.ALIGN_CENTER_VERTICAL|wx.RIGHT|wx.TOP, padding) + + box = wx.BoxSizer(wx.HORIZONTAL) + label = wx.StaticText(self.optionsPage, -1, "When specifying owners and collaborators:", size=(300, -1)) + box.Add(label, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + self.optionsSizer.Add(box, 0, wx.GROW|wx.ALIGN_CENTER_VERTICAL|wx.TOP, padding) + + box = wx.BoxSizer(wx.HORIZONTAL) + self.chkUseEBIUsername = wx.CheckBox(parent=self.optionsPage, id=-1, + label=u'Ignore owners that do not exist', name=u'chkUseEBIUsername', + style=0) + self.chkUseEBIUsername.SetValue(False) + self.chkUseEBIUsername.SetToolTipString(u'Ignore owners that do not exist (for \'Owner\' column datatype)') + box.AddSpacer(wx.Size(indentWidth, -1), border=0, flag=0) + box.Add(self.chkUseEBIUsername, 1, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + self.chkIgnoreNonexistentCollaborators = wx.CheckBox(parent=self.optionsPage, id=-1, + label=u'Ignore collaborators that do not exist', name=u'chkIgnoreNonexistentCollaborators', + style=0) + self.chkIgnoreNonexistentCollaborators.SetValue(False) + self.chkIgnoreNonexistentCollaborators.SetToolTipString(u'Save item even if a specified collaborator does not exist') + box.AddSpacer(wx.Size(indentWidth, -1), border=0, flag=0) + box.Add(self.chkIgnoreNonexistentCollaborators, 1, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + self.optionsSizer.Add(box, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + + box = wx.BoxSizer(wx.HORIZONTAL) + self.chkSaveNonexistentUsernamesAsIDs = wx.CheckBox(parent=self.optionsPage, id=-1, + label=u'Save usernames that are not internal users as user IDs (for LDAP users)', name=u'chkSaveNonexistentUsernamesAsIDs', + style=0) + self.chkSaveNonexistentUsernamesAsIDs.SetValue(False) + self.chkSaveNonexistentUsernamesAsIDs.SetToolTipString(u'Save usernames that are not internal users as user IDs (for LDAP users)') + box.AddSpacer(wx.Size(indentWidth, -1), border=0, flag=0) + box.Add(self.chkSaveNonexistentUsernamesAsIDs, 1, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + self.optionsSizer.Add(box, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + + line = wx.StaticLine(self.optionsPage, -1, size=(20,-1), style=wx.LI_HORIZONTAL) + self.optionsSizer.Add(line, 0, wx.GROW|wx.ALIGN_CENTER_VERTICAL|wx.RIGHT|wx.TOP, padding) + + gridSizer = wx.FlexGridSizer(1, 2) + + box = wx.BoxSizer(wx.HORIZONTAL) + label = wx.StaticText(self.optionsPage, -1, "Export:", size=(300, -1)) + box.Add(label, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + gridSizer.Add(box, 0, wx.GROW|wx.ALIGN_CENTER_VERTICAL|wx.TOP, padding) + + box = wx.BoxSizer(wx.HORIZONTAL) + label = wx.StaticText(self.optionsPage, -1, "WHERE clause:", size=(300, -1)) + box.Add(label, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 0) + gridSizer.Add(box, 0, wx.GROW|wx.ALIGN_CENTER_VERTICAL|wx.TOP, 0) + + leftBox = wx.BoxSizer(wx.VERTICAL) + + box = wx.BoxSizer(wx.HORIZONTAL) + self.chkExport = wx.CheckBox(parent=self.optionsPage, id=-1, + label=u'Export items as CSV', name=u'chkExport', + style=0) + self.chkExport.SetValue(False) + self.chkExport.SetToolTipString(u'Export items as CSV') + box.AddSpacer(wx.Size(indentWidth, -1), border=0, flag=0) + box.Add(self.chkExport, 1, wx.ALIGN_TOP|wx.ALL, padding) + box.AddSpacer(wx.Size(indentWidth, -1), border=0, flag=0) + self.chkIncludeNonLive = wx.CheckBox(parent=self.optionsPage, id=-1, + label=u'Include non-live items', name=u'chkIncludeNonLive', style=0) + self.chkIncludeNonLive.SetValue(False) + self.chkIncludeNonLive.SetToolTipString(u'Include non-live items in export (e.g. archived items)') + box.Add(self.chkIncludeNonLive, 1, wx.ALIGN_TOP|wx.ALL, padding) + + + leftBox.Add(box) + + box = wx.BoxSizer(wx.HORIZONTAL) + box.AddSpacer(wx.Size(indentWidth, -1), border=0, flag=0) + label = wx.StaticText(self.optionsPage, -1, "Filename Conflicts:", size=(-1, -1), style=wx.ALIGN_RIGHT) + box.Add(label, 0, wx.ALIGN_TOP|wx.ALL, padding) + self.cmbConflicts = wx.Choice(choices=[], + id=-1, name=u'cmbConflicts', + parent=self.optionsPage, size=wx.Size(250, 21), style=0) + box.Add(self.cmbConflicts, 1, wx.TOP|wx.ALL, padding) + leftBox.Add(box, 0, wx.ALIGN_TOP|wx.ALL, padding) + + self.cmbConflicts.Append("Do not overwrite any files") + self.cmbConflicts.Append("Overwrite files in target folder") + self.cmbConflicts.Append("Overwrite files with same names") + self.cmbConflicts.SetSelection(0) + + gridSizer.Add(leftBox, 0, wx.TOP|wx.ALL, padding) + + box = wx.BoxSizer(wx.HORIZONTAL) + self.txtWhereClause = wx.TextCtrl(self.optionsPage, -1, "", size=wx.Size(420, 50), style=wx.TE_MULTILINE) + font = wx.Font(10, wx.MODERN, wx.NORMAL, wx.NORMAL, False, u'Courier New') + self.txtWhereClause.SetFont(font) + box.Add(self.txtWhereClause, 1, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 1) + gridSizer.Add(box, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 0) + + self.optionsSizer.Add(gridSizer) + + line = wx.StaticLine(self.optionsPage, -1, size=(20,-1), style=wx.LI_HORIZONTAL) + self.optionsSizer.Add(line, 0, wx.GROW|wx.ALIGN_CENTER_VERTICAL|wx.RIGHT|wx.BOTTOM|wx.TOP, 5) + + # options page Expert script buttons + box = wx.BoxSizer(wx.HORIZONTAL) + label = wx.StaticText(self.optionsPage, -1, "Expert Scripts:", size=(-1, -1)) + box.Add(label, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + self.optionsSizer.Add(box, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 1) + box = wx.BoxSizer(wx.HORIZONTAL) + self.btnStartScript = wx.Button(id=-1, + label=u'Start Script', name=u'btnStartScript', + parent=self.optionsPage, size=wx.Size(132, 23), style=0) + box.Add(self.btnStartScript, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 1) + self.Bind(wx.EVT_BUTTON, self.OnBtnStartScript, self.btnStartScript) + self.btnPreScript = wx.Button(id=-1, + label=u'Row Pre-Script', name=u'btnPreScript', + parent=self.optionsPage, size=wx.Size(132, 23), style=0) + box.Add(self.btnPreScript, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 1) + self.Bind(wx.EVT_BUTTON, self.OnBtnPreScript, self.btnPreScript) + self.btnPostScript = wx.Button(id=-1, + label=u'Row Post-Script', name=u'btnPostScript', + parent=self.optionsPage, size=wx.Size(132, 23), style=0) + box.Add(self.btnPostScript, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 1) + self.Bind(wx.EVT_BUTTON, self.OnBtnPostScript, self.btnPostScript) + self.btnEndScript = wx.Button(id=-1, + label=u'End Script', name=u'btnEndScript', + parent=self.optionsPage, size=wx.Size(132, 23), style=0) + box.Add(self.btnEndScript, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 1) + self.Bind(wx.EVT_BUTTON, self.OnBtnEndScript, self.btnEndScript) + self.optionsSizer.Add(box, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 1) + + self.updateScriptButtonsLabels() + + + # log page + sizer = wx.BoxSizer(orient=wx.VERTICAL) + + box = wx.BoxSizer(orient=wx.HORIZONTAL) + self.btnClearLog = wx.Button(id=-1, label="Clear", parent=self.logPage, size=wx.Size(90, 23), style=0) + self.btnClearLog.SetToolTipString("Clear log") + self.btnClearLog.Bind(wx.EVT_BUTTON, self.OnBtnClearLog) + box.Add(self.btnClearLog, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 4) + sizer.Add(box) + + sizer.Add(self.log, 1, wx.EXPAND) + self.logPage.SetSizer(sizer) + + # import/export buttons + width = 123 + height = 23 + left = 700 + top = 2 + gap = 2 + labelStart = "Start Import" + labelTest = "Test Import" + tooltopStart = "Perform an import/export into EQUELLA" + tooltipTest = "Perform a test run (no items saved in EQUELLA)" + + self.btnConnStartImport = wx.Button(id=-1, label=labelStart, parent=self.connectionPage, pos=wx.Point(left + width + gap, top), size=wx.Size(width, height), style=0) + self.btnConnStartImport.SetToolTipString(tooltopStart) + self.btnConnStartImport.Bind(wx.EVT_BUTTON, self.OnBtnStartImportButton) + + self.btnConnTestImport = wx.Button(id=-1, label=labelTest, parent=self.connectionPage, pos=wx.Point(left, top), size=wx.Size(width, height), style=0) + self.btnConnTestImport.SetToolTipString(tooltipTest) + self.btnConnTestImport.Bind(wx.EVT_BUTTON, self.OnBtnTestImportButton) + + self.btnCsvStartImport = wx.Button(id=-1, label=labelStart, parent=self.csvPage, pos=wx.Point(left + width + gap, top), size=wx.Size(width, height), style=0) + self.btnCsvStartImport.SetToolTipString(tooltopStart) + self.btnCsvStartImport.Bind(wx.EVT_BUTTON, self.OnBtnStartImportButton) + + self.btnCsvTestImport = wx.Button(id=-1, label=labelTest, parent=self.csvPage, pos=wx.Point(left, top), size=wx.Size(width, height), style=0) + self.btnCsvTestImport.SetToolTipString(tooltipTest) + self.btnCsvTestImport.Bind(wx.EVT_BUTTON, self.OnBtnTestImportButton) + + self.btnOptionsStartImport = wx.Button(id=-1, label=labelStart, parent=self.optionsPage, pos=wx.Point(left + width + gap, top), size=wx.Size(width, height), style=0) + self.btnOptionsStartImport.SetToolTipString(tooltopStart) + self.btnOptionsStartImport.Bind(wx.EVT_BUTTON, self.OnBtnStartImportButton) + + self.btnOptionsTestImport = wx.Button(id=-1, label=labelTest, parent=self.optionsPage, pos=wx.Point(left, top), size=wx.Size(width, height), style=0) + self.btnOptionsTestImport.SetToolTipString(tooltipTest) + self.btnOptionsTestImport.Bind(wx.EVT_BUTTON, self.OnBtnTestImportButton) + + self.btnLogStartImport = wx.Button(id=-1, label=labelStart, parent=self.logPage, pos=wx.Point(left + width + gap, top), size=wx.Size(width, height), style=0) + self.btnLogStartImport.SetToolTipString(tooltopStart) + self.btnLogStartImport.Bind(wx.EVT_BUTTON, self.OnBtnStartImportButton) + + self.btnLogTestImport = wx.Button(id=-1, label=labelTest, parent=self.logPage, pos=wx.Point(left, top), size=wx.Size(width, height), style=0) + self.btnLogTestImport.SetToolTipString(tooltipTest) + self.btnLogTestImport.Bind(wx.EVT_BUTTON, self.OnBtnTestImportButton) + + # toolbar and status bar + self.mainStatusBar = wx.StatusBar(id=wxID_MAINFRAMEMAINSTATUSBAR, + name=u'mainStatusBar', parent=self, style=0) + self.mainStatusBar.SetToolTipString(u'Status Bar') + self._init_coll_mainStatusBar_Fields(self.mainStatusBar) + + self.mainToolbar = wx.ToolBar(id=wxID_MAINFRAMEMAINTOOLBAR, + name=u'mainToolbar', parent=self, pos=wx.Point(0, 0), + size=wx.Size(1006, 27), style=wx.TB_HORIZONTAL | wx.NO_BORDER) + + + self._init_coll_mainToolbar_Tools(self.mainToolbar) + + self.progressGauge = wx.Gauge(self.mainStatusBar, -1, style=wx.GA_HORIZONTAL|wx.GA_SMOOTH) + rect = self.mainStatusBar.GetFieldRect(1) + self.progressGauge.SetPosition((rect.x, rect.y)) + self.progressGauge.SetSize((rect.width-40, rect.height)) + + # events to indicate that settings have been changed (and may need to be saved) + self.Bind(wx.EVT_TEXT, self.OnInstitutionUrlChange, self.txtInstitutionUrl) + self.Bind(wx.EVT_CHOICE, self.OnSettingChange, self.cmbEncoding) + self.Bind(wx.EVT_TEXT, self.OnSettingChange, self.txtRowFilter) + self.Bind(wx.EVT_TEXT, self.OnSettingChange, self.txtCSVPath) + self.Bind(wx.EVT_TEXT, self.OnSettingChange, self.txtUsername) + self.Bind(wx.EVT_TEXT, self.OnPasswordChange, self.txtPassword) + self.Bind(wx.EVT_CHOICE, self.OnSettingChange, self.cmbCollections) + self.columnsGrid.Bind(wx.grid.EVT_GRID_CELL_CHANGE, self.OnSettingChange) + self.Bind(wx.EVT_CHECKBOX, self.OnSettingChange, self.chkSaveAsDraft) + self.Bind(wx.EVT_CHECKBOX, self.OnSettingChange, self.chkSaveTestXml) + self.Bind(wx.EVT_CHOICE, self.OnSettingChange, self.cmbExistingMetadataMode) + self.Bind(wx.EVT_CHECKBOX, self.OnSettingChange, self.chkAppendAttachments) + self.Bind(wx.EVT_CHECKBOX, self.OnSettingChange, self.chkCreateNewVersions) + self.Bind(wx.EVT_CHECKBOX, self.OnSettingChange, self.chkUseEBIUsername) + self.Bind(wx.EVT_CHECKBOX, self.OnSettingChange, self.chkIgnoreNonexistentCollaborators) + self.Bind(wx.EVT_CHECKBOX, self.OnSettingChange, self.chkSaveNonexistentUsernamesAsIDs) + self.Bind(wx.EVT_CHECKBOX, self.OnExportChange, self.chkExport) + self.Bind(wx.EVT_CHECKBOX, self.OnExportChange, self.chkIncludeNonLive) + self.Bind(wx.EVT_CHOICE, self.OnSettingChange, self.cmbConflicts) + self.Bind(wx.EVT_TEXT, self.OnSettingChange, self.txtWhereClause) + self.Bind(wx.EVT_TEXT, self.OnSettingChange, self.txtAttachmentsBasepath) + self.Bind(wx.EVT_CLOSE, self.OnClose) + + # accelerator table for Save keyboard shortcuts + randomCtrlSaveId = wx.NewId() + randomCmdSaveId = wx.NewId() + self.Bind(wx.EVT_MENU, self.onCtrlS, id= randomCtrlSaveId) + self.Bind(wx.EVT_MENU, self.onCtrlS, id= randomCmdSaveId) + try: + accel_tbl = wx.AcceleratorTable([(wx.ACCEL_CTRL, ord('S'), randomCtrlSaveId ), + (wx.ACCEL_CMD, ord('S'), randomCmdSaveId )]) + except: + accel_tbl = wx.AcceleratorTable([(wx.ACCEL_CTRL, ord('S'), randomCtrlSaveId )]) + self.SetAcceleratorTable(accel_tbl) + + self._init_sizers() + self.txtInstitutionUrl.SetFocus() + + def OnSettingChange(self, event): + self.dirtyUI = True + event.Skip() + + def OnPasswordChange(self, event): + if self.savePassword: + self.dirtyUI = True + event.Skip() + + def OnExportChange(self, event): + self.UpdateImportExportButtons() + self.dirtyUI = True + event.Skip() + + def OnInstitutionUrlChange(self, event): + self.engine.collectionIDs.clear() + self.engine.eqVersionmm = "" + self.engine.eqVersionmmr = "" + self.engine.eqVersionDisplay = "" + self.dirtyUI = True + event.Skip() + + def UpdateImportExportButtons(self): + if self.chkExport.GetValue(): + self.btnConnStartImport.SetLabel("Start Export") + self.btnConnTestImport.SetLabel("Test Export") + self.btnCsvStartImport.SetLabel("Start Export") + self.btnCsvTestImport.SetLabel("Test Export") + self.btnOptionsStartImport.SetLabel("Start Export") + self.btnOptionsTestImport.SetLabel("Test Export") + self.btnLogStartImport.SetLabel("Start Export") + self.btnLogTestImport.SetLabel("Test Export") + else: + self.btnConnStartImport.SetLabel("Start Import") + self.btnConnTestImport.SetLabel("Test Import") + self.btnCsvStartImport.SetLabel("Start Import") + self.btnCsvTestImport.SetLabel("Test Import") + self.btnOptionsStartImport.SetLabel("Start Import") + self.btnOptionsTestImport.SetLabel("Test Import") + self.btnLogStartImport.SetLabel("Start Import") + self.btnLogTestImport.SetLabel("Test Import") + + def onCtrlS(self, event): + self.mainStatusBar.SetStatusText("Saving...", 0) + self.saveSettings(True) + wx.GetApp().Yield() + self.mainStatusBar.SetStatusText("Ready", 0) + + def getCSVPath(self): + return os.path.join(os.path.dirname(self.settingsfile), self.txtCSVPath.GetValue()) + + def OnBtnBrowseBasePath(self, evt): + dlg = wx.DirDialog(self, "Choose a folder/directory:", + style=wx.DD_DEFAULT_STYLE + ) + + # set the default directory for the dialog + dlg.SetPath(os.path.join(os.path.dirname(self.getCSVPath()), self.txtAttachmentsBasepath.GetValue().strip())) + + if dlg.ShowModal() == wx.ID_OK: + self.txtAttachmentsBasepath.SetValue(dlg.GetPath()) + + # Only destroy a dialog after you're done with it. + dlg.Destroy() + + def OnBtnStartScript(self, evt): + wrapper = [self.startScript] + self.openScriptEditor(wrapper, "Start Script") + self.startScript = wrapper[0] + self.updateScriptButtonsLabels() + + def OnBtnEndScript(self, evt): + wrapper = [self.endScript] + self.openScriptEditor(wrapper, "End Script") + self.endScript = wrapper[0] + self.updateScriptButtonsLabels() + + def OnBtnPreScript(self, evt): + wrapper = [self.preScript] + self.openScriptEditor(wrapper, "Row Pre-Script") + self.preScript = wrapper[0] + self.updateScriptButtonsLabels() + + def OnBtnPostScript(self, evt): + wrapper = [self.postScript] + self.openScriptEditor(wrapper, "Row Post-Script") + self.postScript = wrapper[0] + self.updateScriptButtonsLabels() + + def openScriptEditor(self, script, title): + dlgScript = ScriptDialog(self) + + # get last used dimensions + try: + if self.config.has_option('State', 'scriptdialogsize'): + rawsize = self.config.get('State', 'scriptdialogsize') + size = tuple(int(v) for v in re.findall("[0-9]+", rawsize)) + dlgScript.SetSize(size) + + except: + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + self.engine.echo(''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)), log=False, style=2) + + dlgScript.CenterOnScreen() + dlgScript.SetTitle(title) + dlgScript.scriptEditor.SetText(script[0]) + dlgScript.scriptEditor.EmptyUndoBuffer() + val = dlgScript.ShowModal() + if val == wx.ID_OK: + script[0] = dlgScript.scriptEditor.GetText().replace("\r\n", "\n") + self.dirtyUI = True + + # save dimensions + try: + if not "State" in self.config.sections(): + self.parent.config.add_section('State') + self.config.set('State','scriptdialogsize', str(dlgScript.GetSize())) + self.config.write(open(self.propertiesFile, 'w')) + except: + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + self.engine.echo(''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)), log=False, style=2) + + dlgScript.Destroy() + + def updateScriptButtonsLabels(self): + if self.startScript.strip() != "": + self.btnStartScript.SetLabel("Start Script*") + else: + self.btnStartScript.SetLabel("Start Script") + if self.endScript.strip() != "": + self.btnEndScript.SetLabel("End Script*") + else: + self.btnEndScript.SetLabel("End Script") + if self.preScript.strip() != "": + self.btnPreScript.SetLabel("Row Pre-Script*") + else: + self.btnPreScript.SetLabel("Row Pre-Script") + if self.postScript.strip() != "": + self.btnPostScript.SetLabel("Row Post-Script*") + else: + self.btnPostScript.SetLabel("Row Post-Script") + + def OnBtnClearLog(self, event): + self.ClearLog() + + def ClearLog(self): + self.log.SetReadOnly(False) + self.log.ClearAll() + self.log.AddLogText(self.engine.welcomeLine1 + "\n", 1) + self.log.AddLogText(self.engine.welcomeLine2 + "\n", 1) + + def __init__(self, parent): + # constants + self.METADATA = 'Metadata' + self.ATTACHMENTLOCATIONS = 'Attachment Locations' + self.ATTACHMENTNAMES = 'Attachment Names' + self.CUSTOMATTACHMENTS = 'Custom Attachments' + self.RAWFILES = 'Raw Files' + self.URLS = 'URLs' + self.HYPERLINKNAMES = 'Hyperlink Names' + self.EQUELLARESOURCES = 'EQUELLA Resources' + self.EQUELLARESOURCENAMES = 'EQUELLA Resource Names' + self.COMMANDS = 'Commands' + self.TARGETIDENTIFIER = 'Target Identifier' + self.TARGETVERSION = 'Target Version' + self.COLLECTION = 'Collection' + self.OWNER = 'Owner' + self.COLLABORATORS = 'Collaborators' + self.ITEMID = 'Item ID' + self.ITEMVERSION = 'Item Version' + self.THUMBNAILS = 'Thumbnails' + self.SELECTEDTHUMBNAIL = 'Selected Thumbnail' + self.ROWERROR = 'Row Error' + self.IGNORE = 'Ignore' + + self.COLUMN_POS = "Pos" + self.COLUMN_HEADING = "Column Heading" + self.COLUMN_DATATYPE = "Column Data Type" + self.COLUMN_DISPLAY = "Display" + self.COLUMN_SOURCEIDENTIFIER = "Source Identifier" + self.COLUMN_XMLFRAGMENT = "XML Fragment" + self.COLUMN_DELIMITER = "Delimiter" + + self.OVERWRITENONE = 0 + self.OVERWRITEEXISTING = 1 + self.OVERWRITEALL = 2 + + # globals + self.version = "" + self.debug = False + self.license = "" + self.copyright = "" + self.dirtyUI = False + self.encodingOptions = ["utf-8", "latin1"] + self.startScript = "" + self.endScript = "" + self.preScript = "" + self.postScript = "" + self.clearLogEachRun = False + self.savePassword = True + self.settingsfile = "" + + self._init_ctrls(parent) + rect = self.mainStatusBar.GetFieldRect(1) + self.progressGauge.SetPosition((rect.x, rect.y)) + self.progressBarWidth = rect.width-40 + self.progressBarHeight = rect.height + self.progressGauge.SetSize((0, self.progressBarHeight)) + + self.Center() + + iconFile = os.path.join(self.scriptFolder,"ebibig.ico") + icon = wx.Icon(iconFile, wx.BITMAP_TYPE_ICO) + self.SetIcon(icon) + + # create grid and set column widths and headings + self.columnsGrid.CreateGrid(0, 7) + + self.columnsGrid.SetRowLabelSize(0) + + self.columnsGrid.SetColLabelValue(0, self.COLUMN_POS) + self.columnsGrid.SetColSize(0, 60) + + self.columnsGrid.SetColLabelValue(1, self.COLUMN_HEADING) + self.columnsGrid.SetColSize(1, 360) + + self.columnsGrid.SetColLabelValue(2, self.COLUMN_DATATYPE) + self.columnsGrid.SetColSize(2, 180) + + self.columnsGrid.SetColLabelValue(3, self.COLUMN_DISPLAY) + self.columnsGrid.SetColSize(3, 60) + + self.columnsGrid.SetColLabelValue(4, self.COLUMN_SOURCEIDENTIFIER) + self.columnsGrid.SetColSize(4, 120) + + self.columnsGrid.SetColLabelValue(5, self.COLUMN_XMLFRAGMENT) + self.columnsGrid.SetColSize(5, 110) + + self.columnsGrid.SetColLabelValue(6, self.COLUMN_DELIMITER) + self.columnsGrid.SetColSize(6, 80) + + + # data structures to store column settings + self.currentColumns = [] + self.currentColumnHeadings = [] + + # mouse cursors + self.normalCursor= wx.StockCursor(wx.CURSOR_ARROW) + self.waitCursor= wx.StockCursor(wx.CURSOR_WAIT) + + # main ebi engine + self.engine = None + + self.settingsFile = "" + self.EBIDownloadPage = "" + + self.mainToolbar.EnableTool(wxID_MAINFRAMEMAINTOOLBARSTOP , False) + self.mainToolbar.EnableTool(wxID_MAINFRAMEMAINTOOLBARPAUSE , False) + + self.config = ConfigParser.ConfigParser() + + + + def createEngine(self, version, copyright, license, EBIDownloadPage, propertiesFile): + self.version = version + self.copyright = copyright + self.license = license + self.EBIDownloadPage = EBIDownloadPage + self.propertiesFile = propertiesFile + self.engine = Engine.Engine(self, self.version, self.copyright) + self.engine.setLog(self.log) + + manualConfigPrinted = False + try: + self.config.read(self.propertiesFile) + try: + if self.config.has_option('State', 'mainframesize'): + size = self.config.get('State', 'mainframesize') + self.SetClientSize(tuple(int(v) for v in re.findall("[0-9]+", size))) + self.Center() + except: + pass + + try: + if self.config.has_option('Configuration','attachmentmetadatatargets'): + self.engine.attachmentMetadataTargets = self.config.getboolean('Configuration','attachmentmetadatatargets') + except: + manualConfigPrinted = True + self.engine.echo("Unable to read properties file value 'attachmentmetadatatargets' (reverting to default value): " + str(sys.exc_info()[1])) + try: + if self.config.has_option('Configuration','attachmentchunksize'): + self.engine.chunkSize = self.config.getint('Configuration', 'attachmentchunksize') + except: + manualConfigPrinted = True + self.engine.echo("Unable to read properties file value 'attachmentchunksize' (reverting to default value): " + str(sys.exc_info()[1])) + try: + if self.config.has_option('Configuration','networklogging'): + self.engine.networkLogging = self.config.getboolean('Configuration', 'networklogging') + except: + manualConfigPrinted = True + self.engine.echo("Unable to read properties file value 'networklogging' (reverting to default value): " + str(sys.exc_info()[1])) + try: + if self.config.has_option('Configuration','proxyaddress'): + self.engine.proxy = self.config.get('Configuration', 'proxyaddress') + except: + manualConfigPrinted = True + self.engine.echo("Unable to read properties file value 'proxyaddress': " + str(sys.exc_info()[1])) + try: + if self.config.has_option('Configuration','proxyusername'): + self.engine.proxyUsername = self.config.get('Configuration', 'proxyusername') + except: + manualConfigPrinted = True + self.engine.echo("Unable to read properties file value 'proxyusername': " + str(sys.exc_info()[1])) + try: + if self.config.has_option('Configuration','proxypassword'): + if self.config.get('Configuration', 'proxypassword') != "": + self.engine.proxyPassword = self.dc(self.config.get('Configuration', 'proxypassword')) + else: + self.engine.proxyPassword = "" + except: + manualConfigPrinted = True + self.engine.echo("Unable to read properties file value 'proxyusername': " + str(sys.exc_info()[1])) + try: + if self.config.has_option('Configuration','scormformatsupport'): + self.engine.scormformatsupport = self.config.getboolean('Configuration', 'scormformatsupport') + except: + manualConfigPrinted = True + self.engine.echo("Unable to read properties file value 'scormformatsupport' (reverting to default value): " + str(sys.exc_info()[1])) + try: + if self.config.has_option('Configuration','clearlogeachrun'): + self.clearLogEachRun = self.config.getboolean('Configuration', 'clearlogeachrun') + except: + manualConfigPrinted = True + self.engine.echo("Unable to read properties file value 'clearlogonrun' (reverting to default value): " + str(sys.exc_info()[1])) + try: + if self.config.has_option('Configuration','savepassword'): + self.savePassword = self.config.getboolean('Configuration', 'savepassword') + except: + manualConfigPrinted = True + self.engine.echo("Unable to read properties file value 'savepassword' (reverting to default value): " + str(sys.exc_info()[1])) + try: + if self.config.has_option('Configuration','logbuffersize'): + self.log.logBufferSize = self.config.getint('Configuration', 'logbuffersize') + except: + manualConfigPrinted = True + self.engine.echo("Unable to read properties file value 'logbuffersize' (reverting to default value): " + str(sys.exc_info()[1])) + + except: + self.engine.echo("Unable to read properties file (reverting to default values): " + str(sys.exc_info()[1])) + manualConfigPrinted = True + + if manualConfigPrinted and not self.debug: + self.engine.echo("") + + def printNonDefaultPreferences(self): + manualConfigPrinted = False + if self.debug: + self.engine.echo("debug = True") + manualConfigPrinted = True + if not self.engine.attachmentMetadataTargets: + self.engine.echo("attachmentmetadatatargets = " + str(self.engine.attachmentMetadataTargets)) + manualConfigPrinted = True + if self.engine.chunkSize != self.engine.defaultChunkSize: + self.engine.echo("attachmentchunksize = " + str(self.engine.chunkSize)) + manualConfigPrinted = True + if self.engine.networkLogging: + self.engine.echo("networklogging = " + str(self.engine.networkLogging)) + manualConfigPrinted = True + if not self.engine.scormformatsupport: + self.engine.echo("scormformatsupport = " + str(self.engine.scormformatsupport)) + manualConfigPrinted = True + if manualConfigPrinted: + self.engine.echo("") + + def setVersion(self, version): + self.version = version + + def setDebug(self, debug): + self.debug = debug + + if not self.debug: + try: + self.config.read(self.propertiesFile) + self.debug = self.config.getboolean('Configuration','Debug') + except: + if self.debug: + print str(sys.exc_info()[1]) + + self.engine.setDebug(self.debug) + if self.debug: + self.engine.echo("self.scriptFolder = %s\n" % self.scriptFolder, log=False) + + def OnMainFrameActivate(self, event): + pass + + + def loadCurrentSettings(self): + # connection settings + self.engine.institutionUrl = self.txtInstitutionUrl.GetValue() + if self.txtInstitutionUrl.GetValue().endswith("/"): + self.engine.institutionUrl = self.txtInstitutionUrl.GetValue()[:-1] + self.engine.username = self.txtUsername.GetValue() + self.engine.password = self.txtPassword.GetValue() + self.engine.collection = self.cmbCollections.GetStringSelection() + self.engine.csvFilePath = self.getCSVPath() + self.engine.encoding = self.cmbEncoding.GetStringSelection() + self.engine.rowFilter = self.txtRowFilter.GetValue() + + # options + self.engine.saveAsDraft = self.chkSaveAsDraft.GetValue() + self.engine.saveTestXML = self.chkSaveTestXml.GetValue() + self.engine.existingMetadataMode = self.cmbExistingMetadataMode.GetSelection() + self.engine.appendAttachments = self.chkAppendAttachments.GetValue() + self.engine.createNewVersions = self.chkCreateNewVersions.GetValue() + self.engine.useEBIUsername = self.chkUseEBIUsername.GetValue() + self.engine.ignoreNonexistentCollaborators = self.chkIgnoreNonexistentCollaborators.GetValue() + self.engine.saveNonexistentUsernamesAsIDs = self.chkSaveNonexistentUsernamesAsIDs.GetValue() + self.engine.attachmentsBasepath = self.txtAttachmentsBasepath.GetValue() + self.engine.export = self.chkExport.GetValue() + self.engine.includeNonLive = self.chkIncludeNonLive.GetValue() + self.engine.overwriteMode = self.cmbConflicts.GetSelection() + self.engine.whereClause = self.txtWhereClause.GetValue() + self.engine.startScript = self.startScript + self.engine.endScript = self.endScript + self.engine.preScript = self.preScript + self.engine.postScript = self.postScript + + self.MemorizeColumnSettings() + self.engine.currentColumns = self.currentColumns + self.engine.columnHeadings = self.currentColumnHeadings + + def OnBtnBrowseCSVButton(self, event): + try: + wildcard = "Comma Seperated View (*.csv)|*.csv|All files (*.*)|*.*" + + dlg = wx.FileDialog( + self, message="Select a CSV", + defaultDir=os.getcwd(), + defaultFile="", + wildcard=wildcard, + style=wx.OPEN | wx.CHANGE_DIR + ) + + if dlg.ShowModal() == wx.ID_OK: + paths = dlg.GetPaths() + + self.txtCSVPath.SetValue(paths[0]) + self.LoadCSV() + + dlg.Destroy() + except: + self.SetCursor(self.normalCursor) + self.mainStatusBar.SetStatusText("Error loading CSV", 0) + + errorMessage = self.engine.translateError(str(sys.exc_info()[1])) + + dlg = wx.MessageDialog(None, 'Error loading CSV:\n\n' + errorMessage, 'CSV Load Error', wx.OK | wx.ICON_ERROR) + dlg.ShowModal() + dlg.Destroy() + + self.mainStatusBar.SetStatusText("Ready", 0) + + def unicode_csv_reader(self, utf8_data, encoding, dialect=csv.excel, **kwargs): + csv_reader = csv.reader(utf8_data, dialect=dialect, **kwargs) + firstRow = True + for row in csv_reader: + # remove BOM for utf-8 + if firstRow: + if row[0].startswith(codecs.BOM_UTF8): + row[0] = row[0].decode("utf-8")[1:] + firstRow = False + + yield [cell.decode(encoding) for cell in row] + + def LoadCSV(self): + self.mainStatusBar.SetStatusText("Loading CSV...", 0) + + reader = self.unicode_csv_reader(open(self.getCSVPath(), "rbU"), self.cmbEncoding.GetStringSelection()) + + # store the rows of the CSV in an array + csvArray = [] + for row in reader: + csvArray.append(row) + + self.LoadColumns(csvArray) + plural = "" + if len(csvArray) - 1 != 1: + plural = "s" + self.mainStatusBar.SetStatusText("CSV loaded: %s record%s" % (len(csvArray) - 1, plural), 0) + + + def verifyCurrentColumnsMatchCSV(self): + if self.txtCSVPath.GetValue() != "" and not os.path.isdir(self.txtCSVPath.GetValue()): + reader = self.unicode_csv_reader(open(self.getCSVPath(), "rbU"), self.cmbEncoding.GetStringSelection()) + + # store the first row of the CSV in an array + csvHeadings = [] + for row in reader: + csvHeadings = row + break + + # check if same number of grid headings as there are CSV headings + if len(csvHeadings) != self.columnsGrid.GetNumberRows(): + return False + + # check if all CSV headings match grid headings + i = 0 + for columnHeading in csvHeadings: + if columnHeading.encode(self.cmbEncoding.GetStringSelection()).strip() != self.columnsGrid.GetCellValue(i, 1).encode(self.cmbEncoding.GetStringSelection()): + return False + i += 1 + + return True + + + def OnBtnReloadCSVButton(self, event): + try: + result = wx.ID_YES + columnsAlreadyMatch = self.verifyCurrentColumnsMatchCSV() + if not columnsAlreadyMatch and self.columnsGrid.GetNumberRows() != 0: + dlg = wx.MessageDialog(None, 'CSV headings do not match the current settings.\n\nLoad the CSV and update the current settings?', 'CSV Load', wx.YES_NO | wx.ICON_EXCLAMATION) + result = dlg.ShowModal() + dlg.Destroy() + if result == wx.ID_YES: + self.SetCursor(self.waitCursor) + self.LoadCSV() + self.SetCursor(self.normalCursor) + if columnsAlreadyMatch: + dlg = wx.MessageDialog(None, 'Columns settings correctly match CSV column headings', 'CSV Load', wx.OK | wx.ICON_INFORMATION) + dlg.ShowModal() + dlg.Destroy() + + + except: + self.SetCursor(self.normalCursor) + self.mainStatusBar.SetStatusText("Error loading CSV", 0) + + errorMessage = self.engine.translateError(str(sys.exc_info()[1])) + + dlg = wx.MessageDialog(None, 'Error loading CSV:\n\n' + errorMessage, 'CSV Load Error', wx.OK | wx.ICON_ERROR) + dlg.ShowModal() + dlg.Destroy() + + self.mainStatusBar.SetStatusText("Ready", 0) + + def MemorizeColumnSettings(self): + self.currentColumnHeadings = [] + self.currentColumns = [] + + self.currentColumnDataType = {} + self.currentDisplay = {} + self.currentSourceIdentifier = {} + self.currentXMLFragment = {} + self.currentDelimiter = {} + for row in range(0, self.columnsGrid.GetNumberRows()): + columnHeading = self.columnsGrid.GetCellValue(row, 1) + self.currentColumnHeadings.append(columnHeading) + + columnSettings = {} + columnSettings[self.COLUMN_HEADING] = self.columnsGrid.GetCellValue(row, 1) + columnSettings[self.COLUMN_DATATYPE] = self.columnsGrid.GetCellValue(row, 2) + columnSettings[self.COLUMN_DISPLAY] = self.columnsGrid.GetCellValue(row, 3) + columnSettings[self.COLUMN_SOURCEIDENTIFIER] = self.columnsGrid.GetCellValue(row, 4) + columnSettings[self.COLUMN_XMLFRAGMENT] = self.columnsGrid.GetCellValue(row, 5) + columnSettings[self.COLUMN_DELIMITER] = self.columnsGrid.GetCellValue(row, 6) + self.currentColumns.append(columnSettings) + + + def GetExcelColumnName(self, columnNumber): + dividend = columnNumber + columnName = "" + while dividend > 0: + modulo = (dividend - 1) % 26 + columnName = chr(65 + modulo) + columnName + dividend = (dividend - modulo) / 26 + + return columnName + + def LoadColumns(self, csvArray): + self.MemorizeColumnSettings() + + # delete all grid rows + if self.columnsGrid.GetNumberRows() > 0: + self.columnsGrid.DeleteRows(0, self.columnsGrid.GetNumberRows()) + + # set cell editors + booleanCellEditor = wx.grid.GridCellBoolEditor() + + # try-except for some distros of linux that do not support UseStringValues() + try: + booleanCellEditor.UseStringValues("YES", "") + except: + pass + columnDataTypesCellEditor = wx.grid.GridCellChoiceEditor([self.METADATA, + self.ATTACHMENTLOCATIONS, + self.ATTACHMENTNAMES, + self.CUSTOMATTACHMENTS, + self.RAWFILES, + self.URLS, + self.HYPERLINKNAMES, + self.EQUELLARESOURCES, + self.EQUELLARESOURCENAMES, + self.COMMANDS, + self.TARGETIDENTIFIER, + self.TARGETVERSION, + self.COLLECTION, + self.OWNER, + self.COLLABORATORS, + self.ITEMID, + self.ITEMVERSION, + self.THUMBNAILS, + self.SELECTEDTHUMBNAIL, + self.ROWERROR, + self.IGNORE], False) + + # iterate through columns in CSV + row = 0 + for csvColumnHeadingUnstripped in csvArray[0]: + + # strip the column heading of leading and trailing whitespace + csvColumnHeading = csvColumnHeadingUnstripped.strip() + + colPos = str(row + 1) + "- " + self.GetExcelColumnName(row + 1) + + self.columnsGrid.AppendRows() + self.columnsGrid.SetCellValue(row, 0, colPos) + self.columnsGrid.SetCellValue(row, 1, csvColumnHeading) + + # check if columns already loaded and determine it's occurence if so (may be more than one) + columnPositionLoaded = -1 + occurenceInCsv = csvArray[0][:row + 1].count(csvColumnHeadingUnstripped) + for idx, col in enumerate(self.currentColumnHeadings): + occurenceInCurrentColumns = self.currentColumnHeadings[:idx + 1].count(csvColumnHeading) + if col == csvColumnHeading and occurenceInCurrentColumns == occurenceInCsv: + columnPositionLoaded = idx + + if columnPositionLoaded != -1: + # column already loaded so update with current settings (of same occurence if more than one) + self.columnsGrid.SetCellValue(row, 2, self.currentColumns[columnPositionLoaded][self.COLUMN_DATATYPE]) + self.columnsGrid.SetCellValue(row, 3, self.currentColumns[columnPositionLoaded][self.COLUMN_DISPLAY]) + self.columnsGrid.SetCellValue(row, 4, self.currentColumns[columnPositionLoaded][self.COLUMN_SOURCEIDENTIFIER]) + self.columnsGrid.SetCellValue(row, 5, self.currentColumns[columnPositionLoaded][self.COLUMN_XMLFRAGMENT]) + self.columnsGrid.SetCellValue(row, 6, self.currentColumns[columnPositionLoaded][self.COLUMN_DELIMITER]) + else: + # column not loaded so add to grid as metadata datatype + self.columnsGrid.SetCellValue(row, 2, self.METADATA) + + # set first two cells of each row read-only + self.columnsGrid.SetReadOnly(row, 0) + self.columnsGrid.SetReadOnly(row, 1) + + # set cell alignment + self.columnsGrid.SetCellAlignment(row, 3, wx.ALIGN_CENTER, wx.ALIGN_CENTER) + self.columnsGrid.SetCellAlignment(row, 4, wx.ALIGN_CENTER, wx.ALIGN_CENTER) + self.columnsGrid.SetCellAlignment(row, 5, wx.ALIGN_CENTER, wx.ALIGN_CENTER) + self.columnsGrid.SetCellAlignment(row, 6, wx.ALIGN_CENTER, wx.ALIGN_CENTER) + + # set cell editors (drop-downs) + self.columnsGrid.SetCellEditor(row, 2, columnDataTypesCellEditor) + self.columnsGrid.SetCellEditor(row, 3, booleanCellEditor) + self.columnsGrid.SetCellEditor(row, 4, booleanCellEditor) + self.columnsGrid.SetCellEditor(row, 5, booleanCellEditor) + + self.setCellStates(row) + + row += 1 + + + def OnMainToolbarSaveTool(self, event): + event.Skip() + + # sets cell readonly or editable and sets the background color accordingly + def setCellReadOnly(self, row, col, readonly = True, clearvalue = True): + if readonly: + if clearvalue: + self.columnsGrid.SetCellValue(row, col, "") + self.columnsGrid.SetReadOnly(row, col) + self.columnsGrid.SetCellBackgroundColour(row, col, wx.Colour(230, 230, 240)) + else: + self.columnsGrid.SetReadOnly(row, col, False) + self.columnsGrid.SetCellBackgroundColour(row, col, wx.NullColour) + + # set the grid cell states of a row based on the Column Data Type + def setCellStates(self, row): + if self.columnsGrid.GetCellValue(row, 2) == self.IGNORE: + self.setCellReadOnly(row, 3, False) + self.setCellReadOnly(row, 4, True, False) + self.setCellReadOnly(row, 5, True, False) + self.setCellReadOnly(row, 6, True, False) + else: + if self.columnsGrid.GetCellValue(row, 2) == self.METADATA: + # make all the cells editable + self.setCellReadOnly(row, 3, False) + self.setCellReadOnly(row, 4, False) + self.setCellReadOnly(row, 5, False) + self.setCellReadOnly(row, 6, False) + else: + self.setCellReadOnly(row, 4) + self.setCellReadOnly(row, 5) + + # make Delimiter editable for the following column datatypes + if self.columnsGrid.GetCellValue(row, 2) in [self.METADATA, + self.ATTACHMENTLOCATIONS, + self.ATTACHMENTNAMES, + self.RAWFILES, + self.URLS, + self.HYPERLINKNAMES, + self.EQUELLARESOURCES, + self.EQUELLARESOURCENAMES, + self.THUMBNAILS, + self.COLLABORATORS] and self.columnsGrid.GetCellValue(row, 5) != "YES": + + self.setCellReadOnly(row, 6, False) + else: + self.setCellReadOnly(row, 6) + + # make Display editable for the following column datatypes + if self.columnsGrid.GetCellValue(row, 2) in [self.METADATA, self.IGNORE]: + self.setCellReadOnly(row, 3, False) + else: + self.setCellReadOnly(row, 3) + + def OnColumnsGridGridCellChange(self, event): + row = event.GetRow() + col = event.GetCol() + + + # check if Column Data Type has been changed + if col in [2, 5]: + + self.setCellStates(row) + + # check for any columns Data types that should only have one column selected + if self.columnsGrid.GetCellValue(row, col) in [self.TARGETIDENTIFIER, + self.TARGETVERSION, + self.ITEMID, + self.ITEMVERSION, + self.THUMBNAILS, + self.SELECTEDTHUMBNAIL, + self.ROWERROR, + self.COMMANDS, + self.COLLECTION, + self.OWNER, + self.COLLECTION, + self.COLLABORATORS]: + + for i in range(0, self.columnsGrid.GetNumberRows()): + if i != row and self.columnsGrid.GetCellValue(i, col) == self.columnsGrid.GetCellValue(row, col): + # set any other duplicate settings to Metadata + self.columnsGrid.SetCellValue(i, col, self.METADATA) + self.setCellReadOnly(i, 3, False) + self.setCellReadOnly(i, 4, False) + self.setCellReadOnly(i, 5, False) + self.setCellReadOnly(i, 6, False) + + + # make certain only one column has Source Identifier checked + if col == 4: + for i in range(0, self.columnsGrid.GetNumberRows()): + if i != row: + self.columnsGrid.SetCellValue(i, col, "") + + def ec(self, message): + cm = "" + for i in range(0, len(message)): + cm = cm + '%(#)03d' % {'#':ord(message[i])- i - 1} + if len(message) < 25: + cm = cm + random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ') + for i in range (0, 24 - len(message)): + cm = cm + random.choice('01234567890ABCDEF') + return cm + + def dc(self, cm): + message = "" + i = 0 + while i < len(cm): + try: + message = message + chr(int(cm[i:i + 3]) + i/3 + 1) + except: + break + i = i + 3 + return message + + def saveSettings(self, overwrite = False): + self.SetCursor(self.waitCursor) + + # load column settings to memory + self.MemorizeColumnSettings() + + # create xml document + settingsDoc = Document() + rootNode = settingsDoc.createElement("ebi_settings") + settingsDoc.appendChild(rootNode) + + # create and populate CSV and connection fields + institutionUrlNode = settingsDoc.createElement("institution_url") + if self.txtInstitutionUrl.GetValue().strip() != "": + institutionUrlNode.appendChild(settingsDoc.createTextNode(self.txtInstitutionUrl.GetValue())) + rootNode.appendChild(institutionUrlNode) + + usernameNode = settingsDoc.createElement("username") + if self.txtUsername.GetValue().strip() != "": + usernameNode.appendChild(settingsDoc.createTextNode(self.txtUsername.GetValue())) + rootNode.appendChild(usernameNode) + + if self.savePassword: + passwordNode = settingsDoc.createElement("password") + passwordNode.appendChild(settingsDoc.createTextNode(self.ec(self.txtPassword.GetValue()))) + rootNode.appendChild(passwordNode) + + collectionNode = settingsDoc.createElement("collection") + if self.cmbCollections.GetStringSelection() != "": + collectionNode.appendChild(settingsDoc.createTextNode(self.cmbCollections.GetStringSelection())) + rootNode.appendChild(collectionNode) + + csvNode = settingsDoc.createElement("csv_location") + if self.txtCSVPath.GetValue().strip() != "": + csvNode.appendChild(settingsDoc.createTextNode(self.txtCSVPath.GetValue())) + rootNode.appendChild(csvNode) + + rowFilterNode = settingsDoc.createElement("row_filter") + if self.txtRowFilter.GetValue().strip() != "": + rowFilterNode.appendChild(settingsDoc.createTextNode(self.txtRowFilter.GetValue())) + rootNode.appendChild(rowFilterNode) + + csvEncodingNode = settingsDoc.createElement("csv_encoding") + csvEncodingNode.appendChild(settingsDoc.createTextNode(self.cmbEncoding.GetStringSelection())) + rootNode.appendChild(csvEncodingNode) + + # create and populate columns node + columnsNode = settingsDoc.createElement("columns") + rootNode.appendChild(columnsNode) + for colIndex, csvColumnHeading in enumerate(self.currentColumnHeadings): + columnNode = settingsDoc.createElement("column") + + headingNode = settingsDoc.createElement("heading") + headingNode.appendChild(settingsDoc.createTextNode(csvColumnHeading)) + columnNode.appendChild(headingNode) + + columnDataTypeNode = settingsDoc.createElement("column_data_type") + columnDataTypeNode.appendChild(settingsDoc.createTextNode(self.currentColumns[colIndex][self.COLUMN_DATATYPE])) + columnNode.appendChild(columnDataTypeNode) + + displayNode = settingsDoc.createElement("display") + if self.currentColumns[colIndex][self.COLUMN_DISPLAY].strip() != "": + displayNode.appendChild(settingsDoc.createTextNode(self.currentColumns[colIndex][self.COLUMN_DISPLAY])) + columnNode.appendChild(displayNode) + + sourceIdentifierNode = settingsDoc.createElement("source_identifier") + if self.currentColumns[colIndex][self.COLUMN_SOURCEIDENTIFIER].strip() != "": + sourceIdentifierNode.appendChild(settingsDoc.createTextNode(self.currentColumns[colIndex][self.COLUMN_SOURCEIDENTIFIER])) + columnNode.appendChild(sourceIdentifierNode) + + xmlFragmentNode = settingsDoc.createElement("xml_fragment") + if self.currentColumns[colIndex][self.COLUMN_XMLFRAGMENT].strip() != "": + xmlFragmentNode.appendChild(settingsDoc.createTextNode(self.currentColumns[colIndex][self.COLUMN_XMLFRAGMENT])) + columnNode.appendChild(xmlFragmentNode) + + delimiterNode = settingsDoc.createElement("delimiter") + if self.currentColumns[colIndex][self.COLUMN_DELIMITER].strip() != "": + delimiterNode.appendChild(settingsDoc.createTextNode(self.currentColumns[colIndex][self.COLUMN_DELIMITER])) + columnNode.appendChild(delimiterNode) + + columnsNode.appendChild(columnNode) + + # save Options to xml + saveAsDraftNode = settingsDoc.createElement("save_as_draft") + if self.chkSaveAsDraft.GetValue(): + saveAsDraftNode.appendChild(settingsDoc.createTextNode("True")) + else: + saveAsDraftNode.appendChild(settingsDoc.createTextNode("False")) + rootNode.appendChild(saveAsDraftNode) + + saveTestXmlNode = settingsDoc.createElement("save_test_xml") + if self.chkSaveTestXml.GetValue(): + saveTestXmlNode.appendChild(settingsDoc.createTextNode("True")) + else: + saveTestXmlNode.appendChild(settingsDoc.createTextNode("False")) + rootNode.appendChild(saveTestXmlNode) + + csvExistingMetadataNode = settingsDoc.createElement("existing_metadata_mode") + csvExistingMetadataNode.appendChild(settingsDoc.createTextNode(str(self.cmbExistingMetadataMode.GetSelection()))) + rootNode.appendChild(csvExistingMetadataNode) + + appendAttachmentsNode = settingsDoc.createElement("append_attachments") + if self.chkAppendAttachments.GetValue(): + appendAttachmentsNode.appendChild(settingsDoc.createTextNode("True")) + else: + appendAttachmentsNode.appendChild(settingsDoc.createTextNode("False")) + rootNode.appendChild(appendAttachmentsNode) + + createNewVersionsNode = settingsDoc.createElement("create_new_versions") + if self.chkCreateNewVersions.GetValue(): + createNewVersionsNode.appendChild(settingsDoc.createTextNode("True")) + else: + createNewVersionsNode.appendChild(settingsDoc.createTextNode("False")) + rootNode.appendChild(createNewVersionsNode) + + useEBIUsernameNode = settingsDoc.createElement("use_ebi_username") + if self.chkUseEBIUsername.GetValue(): + useEBIUsernameNode.appendChild(settingsDoc.createTextNode("True")) + else: + useEBIUsernameNode.appendChild(settingsDoc.createTextNode("False")) + rootNode.appendChild(useEBIUsernameNode) + + ignoreNonexistentCollaboratorsNode = settingsDoc.createElement("ignore_nonexistent_collaborators") + if self.chkIgnoreNonexistentCollaborators.GetValue(): + ignoreNonexistentCollaboratorsNode.appendChild(settingsDoc.createTextNode("True")) + else: + ignoreNonexistentCollaboratorsNode.appendChild(settingsDoc.createTextNode("False")) + rootNode.appendChild(ignoreNonexistentCollaboratorsNode) + + saveNonexistentUsernamesAsIDsNode = settingsDoc.createElement("save_nonexistent_usernames_as_ids") + if self.chkSaveNonexistentUsernamesAsIDs.GetValue(): + saveNonexistentUsernamesAsIDsNode.appendChild(settingsDoc.createTextNode("True")) + else: + saveNonexistentUsernamesAsIDsNode.appendChild(settingsDoc.createTextNode("False")) + rootNode.appendChild(saveNonexistentUsernamesAsIDsNode) + + attachmentsBasepathNode = settingsDoc.createElement("attachments_basepath") + if self.txtAttachmentsBasepath.GetValue().strip() != "": + attachmentsBasepathNode.appendChild(settingsDoc.createTextNode(self.txtAttachmentsBasepath.GetValue())) + rootNode.appendChild(attachmentsBasepathNode) + + exportNode = settingsDoc.createElement("export") + if self.chkExport.GetValue(): + exportNode.appendChild(settingsDoc.createTextNode("True")) + else: + exportNode.appendChild(settingsDoc.createTextNode("False")) + rootNode.appendChild(exportNode) + + includeNonLiveNode = settingsDoc.createElement("include_non_live") + if self.chkIncludeNonLive.GetValue(): + includeNonLiveNode.appendChild(settingsDoc.createTextNode("True")) + else: + includeNonLiveNode.appendChild(settingsDoc.createTextNode("False")) + rootNode.appendChild(includeNonLiveNode) + + + csvOverwriteNode = settingsDoc.createElement("overwrite_mode") + csvOverwriteNode.appendChild(settingsDoc.createTextNode(str(self.cmbConflicts.GetSelection()))) + rootNode.appendChild(csvOverwriteNode) + + whereClauseBasepathNode = settingsDoc.createElement("where_clause") + if self.txtWhereClause.GetValue().strip() != "": + whereClauseBasepathNode.appendChild(settingsDoc.createTextNode(self.txtWhereClause.GetValue())) + rootNode.appendChild(whereClauseBasepathNode) + + startScriptNode = settingsDoc.createElement("start_script") + if self.startScript != "": + startScriptNode.appendChild(settingsDoc.createTextNode(self.startScript)) + rootNode.appendChild(startScriptNode) + + endScriptNode = settingsDoc.createElement("end_script") + if self.endScript != "": + endScriptNode.appendChild(settingsDoc.createTextNode(self.endScript)) + rootNode.appendChild(endScriptNode) + + rowPreScriptNode = settingsDoc.createElement("row_pre_script") + if self.preScript != "": + rowPreScriptNode.appendChild(settingsDoc.createTextNode(self.preScript)) + rootNode.appendChild(rowPreScriptNode) + + rowPostScriptNode = settingsDoc.createElement("row_post_script") + if self.postScript != "": + rowPostScriptNode.appendChild(settingsDoc.createTextNode(self.postScript)) + rootNode.appendChild(rowPostScriptNode) + + # get folder from properties file + openDir = os.getcwd() + try: + self.config.read(self.propertiesFile) + openDir = os.path.dirname(self.config.get('State','settingsfile')) + except: + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + self.engine.echo(''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)), log=False, style=2) + + itemSaved = False + filenameSelected = False + path = "" + + # display save dialog if not automatically overwriting existing settings file + if not overwrite or not self.settingsFile.endswith(".ebi"): + + # get default filename + defaultFilename = os.path.basename(self.settingsFile) + if defaultFilename.endswith(".ebi"): + defaultFilename = defaultFilename[:-4] + try: + dlg = wx.FileDialog(self, message="Save settings", + defaultDir=openDir, + defaultFile=defaultFilename, + wildcard="EBI settings file (*.ebi)|*.ebi", + style=wx.SAVE|wx.FD_OVERWRITE_PROMPT) + except: + # catch error caused by Ubuntu wxPython lack of support for wx.FD_OVERWRITE_PROMPT + dlg = wx.FileDialog(self, message="Save settings", + defaultDir=openDir, + defaultFile=defaultFilename, + wildcard="EBI settings file (*.ebi)|*.ebi", + style=wx.SAVE) + if dlg.ShowModal() == wx.ID_OK: + filenameSelected = True + path = dlg.GetPath() + self.settingsfile = path + dlg.Destroy() + else: + path = self.settingsFile + + # write to settings file if one slected or automatically overwriting existing settings file + if filenameSelected or (overwrite and self.settingsFile.endswith(".ebi")): + + # force an extension as macintosh does not add one + if not path.endswith(".ebi"): + path += ".ebi" + + # save settings file as utf-8 + fp = file(path, 'w') + fp.write(settingsDoc.toxml("utf-8")) + fp.close() + + self.settingsFile = path + self.SetTitle('%s - EBI' % os.path.basename(self.settingsFile)) + + self.dirtyUI = False + itemSaved = True + + # save path to properties file + try: + if not "State" in self.config.sections(): + self.config.add_section('State') + self.config.set('State','settingsfile', self.settingsFile) + self.config.write(open(self.propertiesFile, 'w')) + except: + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + self.engine.echo(''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)), log=False, style=2) + time.sleep(0.25) + self.SetCursor(self.normalCursor) + + return itemSaved + + def getText(self, element): + rc = [] + for node in element.childNodes: + if node.nodeType == node.TEXT_NODE: + rc.append(node.data) + return ''.join(rc) + + def loadSettings(self, path): + try: + self.settingsfile = path + + # open settings file as utf-8 + fp = codecs.open(path, 'r', 'utf-8') + settingsString = fp.read() + fp.close() + + # Strip the BOM from the beginning of the Unicode string, if it exists + if settingsString[0] == unicode( codecs.BOM_UTF8, "utf-8" ): + settingsString.lstrip( unicode( codecs.BOM_UTF8, "utf-8" ) ) + + settingsDoc = parseString(settingsString.encode('utf-8')) + + # set connection fields + self.txtInstitutionUrl.SetValue(self.getText(settingsDoc.getElementsByTagName("institution_url")[0])) + self.txtUsername.SetValue(self.getText(settingsDoc.getElementsByTagName("username")[0])) + if len(settingsDoc.getElementsByTagName("password")) > 0: + self.txtPassword.SetValue(self.dc(self.getText(settingsDoc.getElementsByTagName("password")[0]))) + self.txtCSVPath.SetValue(self.getText(settingsDoc.getElementsByTagName("csv_location")[0])) + + # set row filter + if len(settingsDoc.getElementsByTagName("row_filter")) > 0: + self.txtRowFilter.SetValue(self.getText(settingsDoc.getElementsByTagName("row_filter")[0])) + else: + self.txtRowFilter.SetValue("") + + + # load collections drop down + collection = self.getText(settingsDoc.getElementsByTagName("collection")[0]) + self.cmbCollections.Clear() + self.cmbCollections.Append(collection) + self.cmbCollections.SetStringSelection(collection) + + # set CSV encoding option + self.cmbEncoding.SetSelection(self.cmbEncoding.FindString(self.getText(settingsDoc.getElementsByTagName("csv_encoding")[0]))) + + # delete all rows + if self.columnsGrid.GetNumberRows() > 0: + self.columnsGrid.DeleteRows(0, self.columnsGrid.GetNumberRows()) + + # set cell editors + booleanCellEditor = wx.grid.GridCellBoolEditor() + + # try-except for some distros of linux that do not support UseStringValues() + try: + booleanCellEditor.UseStringValues("YES", "") + except: + pass + columnDataTypesCellEditor = wx.grid.GridCellChoiceEditor([self.METADATA, + self.ATTACHMENTLOCATIONS, + self.ATTACHMENTNAMES, + self.CUSTOMATTACHMENTS, + self.RAWFILES, + self.URLS, + self.HYPERLINKNAMES, + self.EQUELLARESOURCES, + self.EQUELLARESOURCENAMES, + self.COMMANDS, + self.TARGETIDENTIFIER, + self.TARGETVERSION, + self.COLLECTION, + self.OWNER, + self.COLLABORATORS, + self.ITEMID, + self.ITEMVERSION, + self.THUMBNAILS, + self.SELECTEDTHUMBNAIL, + self.ROWERROR, + self.IGNORE], False) + + # delete all grid rows + if self.columnsGrid.GetNumberRows() > 0: + self.columnsGrid.DeleteRows(0, self.columnsGrid.GetNumberRows()) + + # populate grid + row = 0 + for columnNode in settingsDoc.getElementsByTagName("columns")[0].getElementsByTagName("column"): + self.columnsGrid.AppendRows() + + colPos = str(row + 1) + "- " + self.GetExcelColumnName(row + 1) + + self.columnsGrid.SetCellValue(row, 0, colPos) + self.columnsGrid.SetCellValue(row, 1, self.getText(columnNode.getElementsByTagName("heading")[0])) + self.columnsGrid.SetCellValue(row, 2, self.getText(columnNode.getElementsByTagName("column_data_type")[0])) + self.columnsGrid.SetCellValue(row, 3, self.getText(columnNode.getElementsByTagName("display")[0])) + self.columnsGrid.SetCellValue(row, 4, self.getText(columnNode.getElementsByTagName("source_identifier")[0])) + self.columnsGrid.SetCellValue(row, 5, self.getText(columnNode.getElementsByTagName("xml_fragment")[0])) + self.columnsGrid.SetCellValue(row, 6, self.getText(columnNode.getElementsByTagName("delimiter")[0])) + + # set cell alignment + self.columnsGrid.SetCellAlignment(row, 3, wx.ALIGN_CENTER, wx.ALIGN_CENTER) + self.columnsGrid.SetCellAlignment(row, 4, wx.ALIGN_CENTER, wx.ALIGN_CENTER) + self.columnsGrid.SetCellAlignment(row, 5, wx.ALIGN_CENTER, wx.ALIGN_CENTER) + self.columnsGrid.SetCellAlignment(row, 6, wx.ALIGN_CENTER, wx.ALIGN_CENTER) + + # set cell editors (drop-downs) + self.columnsGrid.SetCellEditor(row, 2, columnDataTypesCellEditor) + self.columnsGrid.SetCellEditor(row, 3, booleanCellEditor) + self.columnsGrid.SetCellEditor(row, 4, booleanCellEditor) + self.columnsGrid.SetCellEditor(row, 5, booleanCellEditor) + + # set first two cells of each row read-only + self.columnsGrid.SetReadOnly(row, 0) + self.columnsGrid.SetReadOnly(row, 1) + + self.setCellStates(row) + + row += 1 + + # populate Options + + # set save-as-draft option + if len(settingsDoc.getElementsByTagName("save_as_draft")) > 0: + if self.getText(settingsDoc.getElementsByTagName("save_as_draft")[0]) == "True": + self.chkSaveAsDraft.SetValue(True) + else: + self.chkSaveAsDraft.SetValue(False) + else: + self.chkSaveAsDraft.SetValue(False) + + # set Save Test XML option + if len(settingsDoc.getElementsByTagName("save_test_xml")) > 0: + if self.getText(settingsDoc.getElementsByTagName("save_test_xml")[0]) == "True": + self.chkSaveTestXml.SetValue(True) + else: + self.chkSaveTestXml.SetValue(False) + else: + self.chkSaveTestXml.SetValue(False) + + # set existing metadata mode based on deprecated append metadata option + if len(settingsDoc.getElementsByTagName("append_metadata")) > 0: + if self.getText(settingsDoc.getElementsByTagName("append_metadata")[0]) == "True": + self.cmbExistingMetadataMode.SetSelection(2) + else: + self.cmbExistingMetadataMode.SetSelection(0) + else: + self.cmbExistingMetadataMode.SetSelection(0) + + # set existing metadata mode + if len(settingsDoc.getElementsByTagName("existing_metadata_mode")) > 0: + self.cmbExistingMetadataMode.SetSelection(int(self.getText(settingsDoc.getElementsByTagName("existing_metadata_mode")[0]))) + + # set append attachments option + if len(settingsDoc.getElementsByTagName("append_attachments")) > 0: + if self.getText(settingsDoc.getElementsByTagName("append_attachments")[0]) == "True": + self.chkAppendAttachments.SetValue(True) + else: + self.chkAppendAttachments.SetValue(False) + else: + self.chkAppendAttachments.SetValue(False) + + # set create new versions option + if len(settingsDoc.getElementsByTagName("create_new_versions")) > 0: + if self.getText(settingsDoc.getElementsByTagName("create_new_versions")[0]) == "True": + self.chkCreateNewVersions.SetValue(True) + else: + self.chkCreateNewVersions.SetValue(False) + else: + self.chkCreateNewVersions.SetValue(False) + + # set use EBI username option + if len(settingsDoc.getElementsByTagName("use_ebi_username")) > 0: + if self.getText(settingsDoc.getElementsByTagName("use_ebi_username")[0]) == "True": + self.chkUseEBIUsername.SetValue(True) + else: + self.chkUseEBIUsername.SetValue(False) + else: + self.chkUseEBIUsername.SetValue(False) + + # set ignore non-existent collaborators option + if len(settingsDoc.getElementsByTagName("ignore_nonexistent_collaborators")) > 0: + if self.getText(settingsDoc.getElementsByTagName("ignore_nonexistent_collaborators")[0]) == "True": + self.chkIgnoreNonexistentCollaborators.SetValue(True) + else: + self.chkIgnoreNonexistentCollaborators.SetValue(False) + else: + self.chkIgnoreNonexistentCollaborators.SetValue(False) + + # set save non-existent usernames as IDs option + if len(settingsDoc.getElementsByTagName("save_nonexistent_usernames_as_ids")) > 0: + if self.getText(settingsDoc.getElementsByTagName("save_nonexistent_usernames_as_ids")[0]) == "True": + self.chkSaveNonexistentUsernamesAsIDs.SetValue(True) + else: + self.chkSaveNonexistentUsernamesAsIDs.SetValue(False) + else: + self.chkSaveNonexistentUsernamesAsIDs.SetValue(False) + + # set attachments basepath option + if len(settingsDoc.getElementsByTagName("attachments_basepath")) > 0: + self.txtAttachmentsBasepath.SetValue(self.getText(settingsDoc.getElementsByTagName("attachments_basepath")[0])) + else: + self.txtAttachmentsBasepath.SetValue("") + + # set export + if len(settingsDoc.getElementsByTagName("export")) > 0: + if self.getText(settingsDoc.getElementsByTagName("export")[0]) == "True": + self.chkExport.SetValue(True) + else: + self.chkExport.SetValue(False) + else: + self.chkExport.SetValue(False) + + self.UpdateImportExportButtons() + + # set export include non-live + if len(settingsDoc.getElementsByTagName("include_non_live")) > 0: + if self.getText(settingsDoc.getElementsByTagName("include_non_live")[0]) == "True": + self.chkIncludeNonLive.SetValue(True) + else: + self.chkIncludeNonLive.SetValue(False) + else: + self.chkIncludeNonLive.SetValue(False) + + # set overwrite mode + if len(settingsDoc.getElementsByTagName("overwrite_mode")) > 0: + self.cmbConflicts.SetSelection(int(self.getText(settingsDoc.getElementsByTagName("overwrite_mode")[0]))) + + # set where clause + if len(settingsDoc.getElementsByTagName("where_clause")) > 0: + self.txtWhereClause.SetValue(self.getText(settingsDoc.getElementsByTagName("where_clause")[0])) + else: + self.txtWhereClause.SetValue("") + + # set start script + if len(settingsDoc.getElementsByTagName("start_script")) > 0: + self.startScript = self.getText(settingsDoc.getElementsByTagName("start_script")[0]) + else: + self.startScript = "" + + # set end script + if len(settingsDoc.getElementsByTagName("end_script")) > 0: + self.endScript = self.getText(settingsDoc.getElementsByTagName("end_script")[0]) + else: + self.endScript = "" + + # set row pre-script + if len(settingsDoc.getElementsByTagName("row_pre_script")) > 0: + self.preScript = self.getText(settingsDoc.getElementsByTagName("row_pre_script")[0]) + else: + self.preScript = "" + + # set row post-script + if len(settingsDoc.getElementsByTagName("row_post_script")) > 0: + self.postScript = self.getText(settingsDoc.getElementsByTagName("row_post_script")[0]) + else: + self.postScript = "" + + self.updateScriptButtonsLabels() + + self.settingsFile = path + self.SetTitle('%s - EBI' % os.path.basename(self.settingsFile)) + self.mainStatusBar.SetStatusText("Ready", 0) + + self.dirtyUI = False + + return True + + except: + # form error string for debugging + err = str(sys.exc_info()[1]) + errorString = "ERROR opening settings file: " + err + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + if self.debug: + errorString += ': ' + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + self.engine.echo(errorString, log=False, style=2) + self.mainStatusBar.SetStatusText("Ready", 0) + return False + + def OnMainToolbarSaveTool(self, event): + if event.GetId() == wxID_MAINFRAMEMAINTOOLBARSAVE: + # save settings + self.saveSettings() + + if event.GetId() == wxID_MAINFRAMEMAINTOOLBAROPEN: + + # get folder from properties file + openDir = os.getcwd() + try: + self.config.read(self.propertiesFile) + if self.config.has_option("State", 'settingsfile'): + openDir = os.path.dirname(self.config.get('State','settingsfile')) + except: + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + self.engine.echo(''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)), log=False, style=2) + + # open settings file + dlg = wx.FileDialog( + self, message="Open EBI settings file", + defaultDir=openDir, + defaultFile="", + wildcard="EBI settings file (*.ebi)|*.ebi", + style=wx.OPEN | wx.CHANGE_DIR + ) + + if dlg.ShowModal() == wx.ID_OK: + if not self.loadSettings(dlg.GetPaths()[0]): + dlgError = wx.MessageDialog(None, 'Error opening settings file\n\n' + str(sys.exc_info()[1]), 'Settings File Error', wx.OK | wx.ICON_ERROR) + dlgError.ShowModal() + dlgError.Destroy() + + # save path to properties file + try: + if not self.config.has_section("State"): + self.config.add_section('State') + self.config.set('State','settingsfile', self.settingsFile) + self.config.write(open(self.propertiesFile, 'w')) + except: + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + self.engine.echo(''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)), log=False, style=2) + + dlg.Destroy() + + if event.GetId() == wxID_MAINFRAMEMAINTOOLBARSTOP: + # stop processing + self.engine.StopProcessing = True + self.engine.pause = False + self.mainToolbar.EnableTool(wxID_MAINFRAMEMAINTOOLBARSTOP, False) + self.mainToolbar.EnableTool(wxID_MAINFRAMEMAINTOOLBARPAUSE, False) + + if event.GetId() == wxID_MAINFRAMEMAINTOOLBARPAUSE: + # pause/unpause processing + if self.engine.pause: + self.engine.pause = False + else: + self.engine.pause = True + + if event.GetId() == wxID_MAINFRAMEMAINTOOLBAROPTIONS: + dlg = OptionsDialog.OptionsDialog(self) + dlg.CenterOnScreen() + + # populate dialog with preferences + if "Configuration" in self.config.sections() and self.config.getboolean('Configuration','loadlastsettingsfile'): + dlg.chkLoadLastSettingsFile.SetValue(True) + dlg.chkClearLogEachRun.SetValue(self.clearLogEachRun) + dlg.chkSavePassword.SetValue(self.savePassword) + dlg.chkDebugMode.SetValue(self.debug) + dlg.chkNetworkLogging.SetValue(self.engine.networkLogging) + dlg.txtAttachmentsChunkSize.SetValue(str(self.engine.chunkSize)) + dlg.txtLogBufferSize.SetValue(str(self.log.logBufferSize)) + dlg.txtProxyAddress.SetValue(str(self.engine.proxy)) + dlg.txtProxyUsername.SetValue(str(self.engine.proxyUsername)) + dlg.txtProxyPassword.SetValue(str(self.engine.proxyPassword)) + + # this does not return until the dialog is closed. + val = dlg.ShowModal() + + if val == wx.ID_OK: + # set preferences from dialog + + try: + if not "Configuration" in self.config.sections(): + self.config.add_section('Configuration') + + self.config.set('Configuration','loadlastsettingsfile', str(dlg.chkLoadLastSettingsFile.GetValue())) + + self.config.set('Configuration','debug', str(dlg.chkDebugMode.GetValue())) + self.debug = dlg.chkDebugMode.GetValue() + self.engine.debug = dlg.chkDebugMode.GetValue() + + self.config.set('Configuration','clearlogeachrun', str(dlg.chkClearLogEachRun.GetValue())) + self.clearLogEachRun = dlg.chkClearLogEachRun.GetValue() + + self.config.set('Configuration','savepassword', str(dlg.chkSavePassword.GetValue())) + self.savePassword = dlg.chkSavePassword.GetValue() + + self.config.set('Configuration','networklogging', str(dlg.chkNetworkLogging.GetValue())) + self.engine.networkLogging = dlg.chkNetworkLogging.GetValue() + + if dlg.txtAttachmentsChunkSize.GetValue().strip() == "": + self.engine.chunkSize = self.engine.defaultChunkSize + self.config.set('Configuration','attachmentchunksize', self.engine.defaultChunkSize) + else: + try: + self.engine.chunkSize = int(dlg.txtAttachmentsChunkSize.GetValue()) + self.config.set('Configuration','attachmentchunksize', str(dlg.txtAttachmentsChunkSize.GetValue())) + except: + pass + + self.engine.proxy = dlg.txtProxyAddress.GetValue().strip() + self.config.set('Configuration','proxyaddress', self.engine.proxy) + self.engine.proxyUsername = dlg.txtProxyUsername.GetValue().strip() + self.config.set('Configuration','proxyusername', self.engine.proxyUsername) + self.engine.proxyPassword = dlg.txtProxyPassword.GetValue().strip() + if self.engine.proxyPassword != "": + self.config.set('Configuration','proxypassword', self.ec(self.engine.proxyPassword)) + else: + self.config.set('Configuration','proxypassword', "") + + if dlg.txtLogBufferSize.GetValue().strip() == "": + self.log.logBufferSize = self.log.defaultLogBufferSize + self.config.set('Configuration','logbuffersize', self.log.defaultLogBufferSize) + else: + try: + self.log.logBufferSize = int(dlg.txtLogBufferSize.GetValue()) + self.config.set('Configuration','logbuffersize', str(dlg.txtLogBufferSize.GetValue())) + except: + pass + + self.config.write(open(self.propertiesFile, 'w')) + except: + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + self.engine.echo(''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)), log=False, style=2) + + dlg.Destroy() + + if event.GetId() == wxID_MAINFRAMEMAINTOOLBARABOUT: + + # lookup what the latest EBI version is + try: + self.SetCursor(self.waitCursor) + ebiVersion = "" + if self.engine.proxy != "": + + password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() + password_mgr.add_password(None, self.engine.proxy, self.engine.proxyUsername, self.engine.proxyPassword) + proxy_auth_handler = urllib2.ProxyBasicAuthHandler(password_mgr) + proxy_handler = urllib2.ProxyHandler({"http": self.engine.proxy}) + + # build URL opener with proxy + opener = urllib2.build_opener(proxy_handler, proxy_auth_handler) + + f = opener.open(self.EBIDownloadPage + "") + else: + f = urllib2.urlopen(self.EBIDownloadPage + "") + + itemXml = PropBagEx(parseString(f.read())) + itemTitle = itemXml.getNode("xml/item/itembody/name").strip() + ebiVersion = itemTitle[itemTitle.rfind("v") + 1:] + except: + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + self.engine.echo(''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)), log=False, style=2) + + self.SetCursor(self.normalCursor) + + # form latest version text + latestVersionText = "" + if ebiVersion != "": + latestVersionText = "\n(latest version is %s)" % ebiVersion + + # about dialog + info = wx.AboutDialogInfo() + info.Name = "EQUELLA(R) Bulk Importer" + info.Version = self.version + latestVersionText + info.Copyright = self.copyright + info.Description = "" \ + "The EQUELLA Bulk Importer is a program for uploading content into the award winning \n" \ + "EQUELLA(R) content management system.\n\n" \ + "Note that the EQUELLA Bulk Importer is provided \"as-is\". If you wish to have any \n" \ + "extensions made to the program or issues resolved please contact Pearson to engage \n" \ + "the services of an EQUELLA consultant." + info.WebSite = (self.EBIDownloadPage, "Get latest version and documentation") + info.License = self.license + + # display about box + wx.AboutBox(info) + + def populateCollectionsDropDown(self, collectionsList): + selectedCollection = self.cmbCollections.GetStringSelection() + self.cmbCollections.Clear() + if len(collectionsList) == 0: + self.cmbCollections.Append("") + else: + self.cmbCollections.Append("") + for collection in collectionsList: + self.cmbCollections.Append(collection) + self.cmbCollections.SetSelection(0) + if selectedCollection != "" and selectedCollection in self.cmbCollections.GetStrings(): + self.cmbCollections.Delete(0) + self.cmbCollections.SetSelection(self.cmbCollections.FindString(selectedCollection)) + + def OnBtnGetCollectionsButton(self, event): + + self.SetCursor(self.waitCursor) + self.mainStatusBar.SetStatusText("Connecting...", 0) + try: + if self.txtInstitutionUrl.GetValue().strip() == "": + raise Exception, "No insitution URL provided" + + # read current connection parameters + self.loadCurrentSettings() + + # get collection names + collectionsList = self.engine.getContributableCollections() + + # populate collections drop down + self.populateCollectionsDropDown(collectionsList) + + self.mainStatusBar.SetStatusText("Connection successful", 0) + dlg = wx.MessageDialog(None, 'Connection successful and collections retrieved.', 'Connection Successful', wx.OK | wx.ICON_INFORMATION) + dlg.ShowModal() + dlg.Destroy() + self.mainStatusBar.SetStatusText("Ready", 0) + + self.SetCursor(self.normalCursor) + + except: + self.SetCursor(self.normalCursor) + + errorMessage = self.engine.translateError(str(sys.exc_info()[1])) + dlg = wx.MessageDialog(None, 'Error connecting to EQUELLA:\n\n' + errorMessage, 'Connection Error', wx.OK | wx.ICON_ERROR) + self.engine.echo('Error connecting to EQUELLA: ' + errorMessage, style=2) + + # form error string for debugging + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + self.engine.echo("".join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)), log=False, style=2) + + dlg.ShowModal() + dlg.Destroy() + + self.mainStatusBar.SetStatusText("Ready", 0) + + def disableControls(self): + self.btnConnTestImport.Disable() + self.btnConnStartImport.Disable() + self.btnCsvTestImport.Disable() + self.btnCsvStartImport.Disable() + self.btnOptionsTestImport.Disable() + self.btnOptionsStartImport.Disable() + self.btnLogTestImport.Disable() + self.btnLogStartImport.Disable() + self.btnBrowseCSV.Disable() + self.btnReloadCSV.Disable() + self.btnGetCollections.Disable() + self.btnGetCollections.Disable() + self.columnsGrid.Disable() + self.cmbCollections.Disable() + self.txtCSVPath.Disable() + self.txtRowFilter.Disable() + self.cmbEncoding.Disable() + self.txtInstitutionUrl.Disable() + self.txtPassword.Disable() + self.txtUsername.Disable() + self.optionsPage.Disable() + self.btnClearLog.Disable() + self.mainToolbar.EnableTool(wxID_MAINFRAMEMAINTOOLBAROPEN, False) + self.mainToolbar.EnableTool(wxID_MAINFRAMEMAINTOOLBARSAVE , False) + self.mainToolbar.EnableTool(wxID_MAINFRAMEMAINTOOLBAROPTIONS , False) + + def enableControls(self): + self.btnConnTestImport.Enable() + self.btnConnStartImport.Enable() + self.btnCsvTestImport.Enable() + self.btnCsvStartImport.Enable() + self.btnOptionsTestImport.Enable() + self.btnOptionsStartImport.Enable() + self.btnLogTestImport.Enable() + self.btnLogStartImport.Enable() + self.btnBrowseCSV.Enable() + self.btnReloadCSV.Enable() + self.btnGetCollections.Enable() + self.btnGetCollections.Enable() + self.columnsGrid.Enable() + self.cmbCollections.Enable() + self.txtCSVPath.Enable() + self.txtRowFilter.Enable() + self.cmbEncoding.Enable() + self.txtInstitutionUrl.Enable() + self.txtPassword.Enable() + self.txtUsername.Enable() + self.optionsPage.Enable() + self.btnClearLog.Enable() + self.mainToolbar.EnableTool(wxID_MAINFRAMEMAINTOOLBAROPEN, True) + self.mainToolbar.EnableTool(wxID_MAINFRAMEMAINTOOLBARSAVE , True) + self.mainToolbar.EnableTool(wxID_MAINFRAMEMAINTOOLBAROPTIONS , True) + + def OnBtnStartImportButton(self, event): + self.startImport(testOnly = False) + + def OnBtnTestImportButton(self, event): + self.startImport(testOnly = True) + + def startImport(self, testOnly): + try: + # Mac bug workaround: commit any changed but uncommitted cells in wxGrid + self.columnsGrid.SetGridCursor(self.columnsGrid.GetGridCursorRow(), self.columnsGrid.GetGridCursorCol()) + except: + pass + try: + self.progressGauge.SetSize((self.progressBarWidth, self.progressBarHeight)) + self.progressGauge.Hide() + + if self.txtInstitutionUrl.GetValue().strip() == "": + raise Exception, "No institution URL provided" + + if self.cmbCollections.GetStringSelection() == "" or self.cmbCollections.GetStringSelection() == "": + try: + self.loadCurrentSettings() + self.populateCollectionsDropDown(sorted(self.engine.getContributableCollections())) + except: + pass + raise Exception, "No collection selected" + + if self.getCSVPath() == "": + raise Exception, "No CSV selected" + + + + # clear log if required + if self.clearLogEachRun: + self.ClearLog() + + if self.verifyCurrentColumnsMatchCSV(): + self.disableControls() + self.mainToolbar.EnableTool(wxID_MAINFRAMEMAINTOOLBARSTOP , True) + self.mainToolbar.EnableTool(wxID_MAINFRAMEMAINTOOLBARPAUSE , True) + + # select log page (tab) + self.nb.SetSelection(3) + + # read current connection parameters and run import + self.loadCurrentSettings() + + # print preferences + self.printNonDefaultPreferences() + + # perform run + self.log.Disable() + self.engine.runImport(self, testOnly) + self.log.Enable() + + self.enableControls() + self.mainToolbar.EnableTool(wxID_MAINFRAMEMAINTOOLBARSTOP , False) + self.mainToolbar.EnableTool(wxID_MAINFRAMEMAINTOOLBARPAUSE , False) + + self.progressGauge.SetSize((0, self.progressBarHeight)) + self.progressGauge.SetValue(0) + + else: + # select csv page (tab) + self.nb.SetSelection(1) + + dlg = wx.MessageDialog(None, 'CSV headings do not match the column headings in the settings.\n\nThe CSV headings may have been changed. Try reloading the CSV or modify the CSV to match the settings.', 'CSV Load Error', wx.OK | wx.ICON_ERROR) + dlg.ShowModal() + dlg.Destroy() + + # (re)populate collections drop down + if len(self.engine.collectionIDs) > 0: + self.populateCollectionsDropDown(sorted(self.engine.collectionIDs.keys())) + + except: + self.progressGauge.Hide() + + # select log page (tab) + self.nb.SetSelection(3) + self.log.Enable() + + # form error string for debugging + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + errorString = "ERROR: " + str(exceptionValue) + if self.debug: + errorString += ': ' + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + self.engine.echo(errorString, log=False, style=2) + self.enableControls() + self.mainToolbar.EnableTool(wxID_MAINFRAMEMAINTOOLBARSTOP, False) + self.mainToolbar.EnableTool(wxID_MAINFRAMEMAINTOOLBARPAUSE, False) + + def OnClose(self, event): + # save main frame dimensions to properties file + try: + if not "State" in self.config.sections(): + self.config.add_section('State') + self.config.set('State','mainframesize', self.GetClientSize()) + self.config.write(open(self.propertiesFile, 'w')) + except: + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + self.engine.echo(''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)), log=False, style=2) + + if self.dirtyUI: + dlg = wx.MessageDialog(self, + "Do you want to save your settings before exiting?", + "Save Settings", + wx.YES|wx.NO|wx.CANCEL|wx.ICON_EXCLAMATION) + result = dlg.ShowModal() + dlg.Destroy() + if result == wx.ID_NO: + self.Destroy() + if result == wx.ID_YES: + # save settings + if self.saveSettings(True): + self.Destroy() + else: + self.Destroy() + +class Log(stc.StyledTextCtrl): + def __init__(self, parent, ID=-1, pos=wx.DefaultPosition, size=wx.DefaultSize, style=0): + stc.StyledTextCtrl.__init__(self, parent, ID, pos, size, style) + self.parent = parent + self.defaultLogBufferSize = 1000 + self.logBufferSize = self.defaultLogBufferSize + self.logBufferChunk = 100 + self.SetReadOnly(True) + + if wx.Platform == '__WXMSW__': + faces = { 'mono' : 'Courier New', + 'size' : 10, + } + elif wx.Platform == '__WXMAC__': + faces = { 'mono' : 'Monaco', + 'size' : 12, + } + else: + faces = { 'mono' : 'Courier', + 'size' : 12, + } + + # default style + self.StyleSetSpec(stc.STC_STYLE_DEFAULT, "fore:#0000FF,face:%(mono)s,size:%(size)d" % faces) + self.StyleClearAll() + self.StyleSetSpec(1, "fore:#000000") + self.StyleSetSpec(2, "fore:#ff0000") + self.StyleSetSpec(3, "fore:#009900") + self.SetWrapMode(True) + + def AddLogText(self, text, style = 0): + self.SetReadOnly(False) + self.AppendText(text.decode(self.parent.owner.cmbEncoding.GetStringSelection())) + if style != 0: + self.StartStyling(self.GetTextLength() - len(text), 0xff) + self.SetStyling(len(text) - 1, style) + if self.GetLineCount() > self.logBufferSize + self.logBufferChunk: + self.SetSelection(0, self.PositionFromLine(self.GetLineCount() - self.logBufferSize)) + self.DeleteBack() + + self.SetReadOnly(True) + self.GotoPos(self.GetLength()) + + +class ScriptEditor(stc.StyledTextCtrl): + + def __init__(self, parent, ID=-1, pos=wx.DefaultPosition, size=wx.DefaultSize, style=0): + stc.StyledTextCtrl.__init__(self, parent, ID, pos, size, style) + + self.SetLexer(stc.STC_LEX_PYTHON) + ebiScriptKeywords = [ + "IMPORT", + "EXPORT", + "NEWITEM", + "NEWVERSION", + "EDITITEM", + "DELETEITEM", + "mode", + "action", + "vars", + "rowData", + "rowCounter", + "testOnly", + "institutionUrl", + "collection", + "username", + "logger", + "columnHeadings", + "columnSettings", + "successCount", + "errorCount", + "itemId", + "itemVersion", + "xml", + "xmldom", + "process", + "csvData", + "True", + "False", + "sourceIdentifierIndex", + "targetIdentifierIndex", + "targetVersionIndex", + "imsmanifest", + "ebi", + "equella", + ] + self.SetKeyWords(0, " ".join(keyword.kwlist) + " " + " ".join(ebiScriptKeywords)) + + self.SetMarginWidth(1,40) + self.SetMarginWidth(2,5) + self.SetMarginType(1, wx.stc.STC_MARGIN_NUMBER) + self.SetViewWhiteSpace(False) + self.SetEdgeMode(stc.STC_EDGE_BACKGROUND) + self.SetEdgeColumn(160) + + if wx.Platform == '__WXMSW__': + faces = { 'times': 'Times New Roman', + 'mono' : 'Courier New', + 'helv' : 'Arial', + 'other': 'Comic Sans MS', + 'size' : 10, + 'size2': 8, + } + elif wx.Platform == '__WXMAC__': + faces = { 'times': 'Times New Roman', + 'mono' : 'Monaco', + 'helv' : 'Arial', + 'other': 'Comic Sans MS', + 'size' : 12, + 'size2': 10, + } + else: + faces = { 'times': 'Times', + 'mono' : 'Courier', + 'helv' : 'Helvetica', + 'other': 'new century schoolbook', + 'size' : 12, + 'size2': 10, + } + + # Global default styles for all languages + self.StyleSetSpec(stc.STC_STYLE_DEFAULT, "face:%(mono)s,size:%(size)d" % faces) + self.StyleClearAll() # Reset all to be like the default + self.StyleSetSpec(stc.STC_STYLE_LINENUMBER, "back:#C0C0C0,face:%(helv)s,size:%(size2)d" % faces) + self.StyleSetSpec(stc.STC_STYLE_CONTROLCHAR, "face:%(other)s" % faces) + self.StyleSetSpec(stc.STC_STYLE_BRACELIGHT, "fore:#FFFFFF,back:#0000FF,bold") + self.StyleSetSpec(stc.STC_STYLE_BRACEBAD, "fore:#000000,back:#FF0000,bold") + + # Python styles + # Default + self.StyleSetSpec(stc.STC_P_DEFAULT, "fore:#000000,face:%(mono)s,size:%(size)d" % faces) + # Comments + self.StyleSetSpec(stc.STC_P_COMMENTLINE, "fore:#007F00,face:%(mono)s,size:%(size)d" % faces) + # Number + self.StyleSetSpec(stc.STC_P_NUMBER, "fore:#007F7F,size:%(size)d" % faces) + # String + self.StyleSetSpec(stc.STC_P_STRING, "fore:#7F007F,face:%(mono)s,size:%(size)d" % faces) + # Single quoted string + self.StyleSetSpec(stc.STC_P_CHARACTER, "fore:#7F007F,face:%(mono)s,size:%(size)d" % faces) + # Keyword + self.StyleSetSpec(stc.STC_P_WORD, "fore:#00007F,bold,size:%(size)d" % faces) + # Triple quotes + self.StyleSetSpec(stc.STC_P_TRIPLE, "fore:#7F0000,size:%(size)d" % faces) + # Triple double quotes + self.StyleSetSpec(stc.STC_P_TRIPLEDOUBLE, "fore:#7F0000,size:%(size)d" % faces) + # Class name definition + self.StyleSetSpec(stc.STC_P_CLASSNAME, "fore:#0000FF,bold,underline,size:%(size)d" % faces) + # Function or method name definition + self.StyleSetSpec(stc.STC_P_DEFNAME, "fore:#007F7F,bold,size:%(size)d" % faces) + # Operators + self.StyleSetSpec(stc.STC_P_OPERATOR, "bold,size:%(size)d" % faces) + # Identifiers + self.StyleSetSpec(stc.STC_P_IDENTIFIER, "fore:#000000,face:%(mono)s,size:%(size)d" % faces) + # Comment-blocks + self.StyleSetSpec(stc.STC_P_COMMENTBLOCK, "fore:#7F7F7F,size:%(size)d" % faces) + # End of line where string is not closed + self.StyleSetSpec(stc.STC_P_STRINGEOL, "fore:#000000,face:%(mono)s,back:#E0C0E0,eol,size:%(size)d" % faces) + + self.SetCaretForeground("BLUE") + + self.SetUseTabs(0) + self.SetTabWidth(4) + self.SetTabIndents(1) + self.SetBackSpaceUnIndents(1) + + self.Bind(wx.EVT_KEY_UP, self.OnKeyUp) + + def OnKeyUp(self,event): + key = event.GetKeyCode() + + # auto indent + if key == wx.WXK_NUMPAD_ENTER or key == wx.WXK_RETURN: + line = self.GetCurrentLine() + indentWidth = self.GetLineIndentation(line - 1) + + # add extra indent if following an indent keyword + indentKeywords = ['if ','else:','elif ','for ','while ','def ','class ','try:','except ','finally:'] + if filter(self.GetLineRaw(line - 1).strip().startswith,indentKeywords + [''])[0] in indentKeywords: + indentWidth += self.GetTabWidth() + + self.SetLineIndentation(line, indentWidth) + for i in range(indentWidth): + self.CharRight() + + event.Skip() + +class ScriptDialog(wx.Dialog): + + def __init__(self, prnt): + wx.Dialog.__init__(self, style=wx.DEFAULT_DIALOG_STYLE|wx.RESIZE_BORDER, parent=prnt, name='', title='Script', pos=wx.Point(-1, -1), id=-1, size=wx.Size(700, 600)) + + self.scriptEditor = ScriptEditor(self) + + sizer = wx.BoxSizer(wx.VERTICAL) + sizer.Add(self.scriptEditor, 1, wx.EXPAND) + + btnsizer = wx.StdDialogButtonSizer() + btn = wx.Button(self, wx.ID_OK) + btn.SetDefault() + btnsizer.AddButton(btn) + btn = wx.Button(self, wx.ID_CANCEL) + btnsizer.AddButton(btn) + btnsizer.Realize() + sizer.Add(btnsizer, 0, wx.ALIGN_RIGHT|wx.RIGHT|wx.ALL, 5) + + self.SetSizer(sizer) diff --git a/package-mac/source/ebi/OptionsDialog.py b/package-mac/source/ebi/OptionsDialog.py new file mode 100644 index 0000000..a1a1a68 --- /dev/null +++ b/package-mac/source/ebi/OptionsDialog.py @@ -0,0 +1,154 @@ +#Boa:Dialog:OptionsDialog + +# Author: Jim Kurian, Pearson plc. +# Date: October 2014 +# +# EBI Preferences dialog. Requires MainFrame.py to launch it. + +import sys, wx, os, keyword, re +import wx.stc as stc + +def create(parent): + return OptionsDialog(parent) + +[wxID_OPTIONSDIALOG] = [wx.NewId() for _init_ctrls in range(1)] + +class BasicPage(wx.Panel): + def __init__(self, parent): + wx.Panel.__init__(self, parent) + +class AdvancedPage(wx.Panel): + def __init__(self, parent): + wx.Panel.__init__(self, parent) + +class OptionsDialog(wx.Dialog): + def _init_ctrls(self, prnt): + wx.Dialog.__init__(self, style=wx.DEFAULT_DIALOG_STYLE, name='', parent=prnt, title='Preferences', pos=wx.Point(-1, -1), id=wxID_OPTIONSDIALOG, size=wx.Size(440, 350)) + + # notebook (tabs) + self.nb = wx.Notebook(parent=self, pos=wx.Point(0, 47)) + self.basicPage = BasicPage(self.nb) + self.nb.AddPage(self.basicPage, "Basic") + self.advancedPage = AdvancedPage(self.nb) + self.nb.AddPage(self.advancedPage, "Advanced") + + self.mainBoxSizer = wx.BoxSizer(orient=wx.VERTICAL) + self.mainBoxSizer.AddWindow(self.nb, 1, border=0, flag=wx.EXPAND) + self.SetSizer(self.mainBoxSizer) + + padding = 5 + + sizer = wx.BoxSizer(wx.VERTICAL) + + sizer.AddSpacer(10) + + box = wx.BoxSizer(wx.HORIZONTAL) + self.chkLoadLastSettingsFile = wx.CheckBox(parent=self.basicPage, id=-1, label=u'Load last settings file when starting EBI', style=0) + self.chkLoadLastSettingsFile.SetValue(False) + box.Add(self.chkLoadLastSettingsFile, 1, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + sizer.Add(box, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + + box = wx.BoxSizer(wx.HORIZONTAL) + self.chkClearLogEachRun = wx.CheckBox(parent=self.basicPage, id=-1, label="Clear log each run", style=0) + self.chkClearLogEachRun.SetValue(False) + self.chkClearLogEachRun.SetToolTipString("Clear log at the start of each run") + box.Add(self.chkClearLogEachRun, 1, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + sizer.Add(box, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + + box = wx.BoxSizer(wx.HORIZONTAL) + self.chkSavePassword = wx.CheckBox(parent=self.basicPage, id=-1, label="Save password in settings file", style=0) + self.chkSavePassword.SetValue(True) + self.chkSavePassword.SetToolTipString("Save password in settings file") + box.Add(self.chkSavePassword, 1, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + sizer.Add(box, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + + sizer.AddSpacer(3) + + box = wx.BoxSizer(wx.HORIZONTAL) + label = wx.StaticText(self.basicPage, -1, "Log buffer size (lines):") + box.Add(label, 0, wx.ALIGN_RIGHT|wx.ALL, padding) + self.txtLogBufferSize = wx.TextCtrl(self.basicPage, -1, "", size=wx.Size(60,-1)) + box.Add(self.txtLogBufferSize) + sizer.Add(box, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + + self.basicPage.SetSizer(sizer) + + sizer = wx.BoxSizer(wx.VERTICAL) + + sizer.AddSpacer(10) + + + gridSizer = wx.FlexGridSizer(1, 2) + + label = wx.StaticText(id=-1, label=u'Proxy Server Address:', parent=self.advancedPage, style=wx.ALIGN_RIGHT) + gridSizer.Add(label, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 4) + self.txtProxyAddress = wx.TextCtrl(id=-1, + name=u'txtProxyAddress', parent=self.advancedPage, + size=wx.Size(280, 21), style=0, value=u'') + self.txtProxyAddress.SetToolTipString(u'Proxy address e.g. "delphi_proxy:8080"') + gridSizer.Add(self.txtProxyAddress) + + gridSizer.AddSpacer(5) + gridSizer.AddSpacer(5) + + label = wx.StaticText(id=-1, label=u'Proxy Server Username:', parent=self.advancedPage, style=wx.ALIGN_RIGHT) + gridSizer.Add(label, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 4) + self.txtProxyUsername = wx.TextCtrl(id=-1, + name=u'txtProxyUsername', parent=self.advancedPage, + size=wx.Size(150, 21), style=0) + gridSizer.Add(self.txtProxyUsername) + + gridSizer.AddSpacer(5) + gridSizer.AddSpacer(5) + + label = wx.StaticText(id=-1, label=u'Proxy Server Password:', parent=self.advancedPage, style=wx.ALIGN_RIGHT) + gridSizer.Add(label, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 4) + self.txtProxyPassword = wx.TextCtrl(id=-1, + name=u'txtProxyPassword', parent=self.advancedPage, + size=wx.Size(150, 21), style=wx.TE_PASSWORD) + gridSizer.Add(self.txtProxyPassword) + + sizer.Add(gridSizer, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + + sizer.AddSpacer(15) + + box = wx.BoxSizer(wx.HORIZONTAL) + self.chkDebugMode = wx.CheckBox(parent=self.advancedPage, id=-1, label=u'Debug Mode', style=0) + self.chkDebugMode.SetValue(False) + box.Add(self.chkDebugMode, 1, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + sizer.Add(box, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + + box = wx.BoxSizer(wx.HORIZONTAL) + self.chkNetworkLogging = wx.CheckBox(parent=self.advancedPage, id=-1, label=u'Network Logging (Warning: only use for small runs)', style=0) + self.chkNetworkLogging.SetValue(False) + box.Add(self.chkNetworkLogging, 1, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + sizer.Add(box, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + + sizer.AddSpacer(3) + + box = wx.BoxSizer(wx.HORIZONTAL) + label = wx.StaticText(self.advancedPage, -1, "Attachment chunk size (bytes):") + box.Add(label, 0, wx.ALIGN_RIGHT|wx.ALL, padding) + self.txtAttachmentsChunkSize = wx.TextCtrl(self.advancedPage, -1, "", size=wx.Size(80,-1)) + box.Add(self.txtAttachmentsChunkSize) + sizer.Add(box, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, padding) + + self.advancedPage.SetSizer(sizer) + + btnsizer = wx.StdDialogButtonSizer() + btn = wx.Button(self, wx.ID_OK) + btn.SetDefault() + btnsizer.AddButton(btn) + btn = wx.Button(self, wx.ID_CANCEL) + btnsizer.AddButton(btn) + btnsizer.Realize() + self.mainBoxSizer.Add(btnsizer, 0, wx.ALIGN_RIGHT|wx.RIGHT|wx.ALL, 5) + + + def __init__(self, parent): + self.startScript = "" + self.endScript = "" + self.preScript = "" + self.postScript = "" + self._init_ctrls(parent) + diff --git a/package-mac/source/ebi/ebi.properties b/package-mac/source/ebi/ebi.properties new file mode 100644 index 0000000..1241da6 --- /dev/null +++ b/package-mac/source/ebi/ebi.properties @@ -0,0 +1,6 @@ +[Configuration] +loadlastsettingsfile = False + +[State] +mainframesize = (1012, 575) + diff --git a/package-mac/source/ebi/ebi.py b/package-mac/source/ebi/ebi.py new file mode 100644 index 0000000..35ec9b1 --- /dev/null +++ b/package-mac/source/ebi/ebi.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python +#Boa:App:ebi + +# EQUELLA BULK IMPORTER (EBI) +# Author: Jim Kurian, Pearson plc. +# +# This program creates items in EQUELLA based on metadata specified in a csv file. The csv file +# must commence with a row of metadata xpaths (e.g. metadata/name,metadata/description,...). +# Subsequent rows should contain the metadata that populate the element identified by the +# xpath in the column header. +# +# metadata/title,metadata/description +# Our House,"This is a picture of my house, my lawn, my cat and my dog" +# Our Car,This is a picture of my car +# +# User Guide +# A detailed and comprehensive user guide is distributed with this program. Please see this for more +# detailed usage instructions and troubleshooting tips. + +# system settings (do not change!) +Version = "4.71" +Copyright = "Copyright (c) 2014 Pearson plc. All rights reserved." +License = """ +THE EQUELLA(R) BULK IMPORTER PROGRAM IS PROVIDED UNDER THE TERMS OF THIS LICENSE ("AGREEMENT"). ANY USE OF THE PROGRAM CONSTITUTES THE RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS +"Vendor" means Pearson plc. and its subsidiaries worldwide. + +"Program" means all versions of the EQUELLA Bulk Importer software program, a software program designed for managing data in the EQUELLA(R) software program. + +"Recipient" means anyone who receives the Program under this Agreement. + +2. NO WARRANTY + +THE PROGRAM IS PROVIDED ON AN "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT OR FITNESS FOR A PARTICULAR PURPOSE. +The Recipient is solely responsible for determining the appropriateness of using the Program and assumes all risks associated with its exercise of rights under this Agreement, including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and unavailability or interruption of operations. + +3. USAGE AND DISTRIBUTION + +The Recipient of the Program may use it for the management of the EQUELLA software program within the Recipient's organization only. The Recipient may only distribute the Program within the Recipient's organization. +""" +EBIDownloadPage = "http://maestro.equella.com/items/eb737eb2-ac6f-4ba3-af17-321ee6c305a1/0/" +Debug = False +SuppressVersion = False + +import sys, os, traceback +import wx +import MainFrame +import ConfigParser +import platform + +modules ={'MainFrame': [1, u'Main frame of EBI', 'none://MainFrame.py']} + +propertiesFile = "ebi.properties" +if sys.path[0].endswith(".zip"): + propertiesFile = os.path.join(os.path.dirname(sys.path[0]), propertiesFile) +else: + propertiesFile = os.path.join(sys.path[0], propertiesFile) + +display = True + +class ebi(wx.App): + global display + + def OnInit(self): + self.main = MainFrame.create(None) + if display: + self.main.Show() + self.SetTopWindow(self.main) + return True + +def alert(message): + message = message + "\n\n" + Copyright + app = wx.PySimpleApp() + dialog = wx.Dialog(None, title='EQUELLA Bulk Importer ' + Version, size=wx.Size(500, 300), style=wx.RESIZE_BORDER|wx.DEFAULT_DIALOG_STYLE) + dialog.Center() + box = wx.BoxSizer(wx.VERTICAL) + dialog.SetSizer(box) + txtMessage = wx.TextCtrl(dialog, -1, message, style=wx.TE_MULTILINE|wx.TE_READONLY) + box.Add(txtMessage, 1, wx.ALIGN_CENTER|wx.ALL|wx.EXPAND) + + btnsizer = wx.StdDialogButtonSizer() + btn = wx.Button(dialog, wx.ID_OK) + btn.SetDefault() + btnsizer.AddButton(btn) + btnsizer.Realize() + box.Add(btnsizer, 0, wx.ALIGN_CENTER|wx.CENTER|wx.ALL, 5) + btn.SetFocus() + + dialog.ShowModal() + dialog.Destroy() + app.MainLoop() + + +def main(): + try: + global display + global Version + global Debug + global SuppressVersion + global propertiesFile + + # create properties file + config = ConfigParser.ConfigParser() + config.read(propertiesFile) + if not "Configuration" in config.sections(): + config.add_section('Configuration') + config.set('Configuration','LoadLastSettingsFile', 'False') + config.write(open(propertiesFile, 'w')) + else: + try: + if config.has_option('Configuration','debug'): + Debug = config.getboolean('Configuration','debug') + except: + alert("Error reading properties file for debug setting: " + str(sys.exc_info()[1])) + + # usage syntax to display in command line + usageSyntax = """USAGE: + +ebi.py [-start] [-test] [] +ebi.exe [-start] [-test] [] + + +Run the EBI visually and load the specified settings file. + +-start +Run the EBI non-visually using the specified settings file. + +-test +Run the EBI non-visually in test mode using the specified settings file, no items will be submitted to EQUELLA. + """ + + settingsFile = "" + + if "?" in sys.argv: + alert(usageSyntax) + + else: + usageCorrect = True + i = 1 + + #loop through arguments (skip first argument as it is the command itself) + while i < len(sys.argv): + if sys.argv[i] in ["-test", "-start"]: + + # check that an argument exists after the -start or -test argument + if i + 1 <= (len(sys.argv) - 1): + + # check that argument after -settings does not start with a dash + if sys.argv[i + 1][0] != "-": + + # get settings filename + settingsFile = sys.argv[i + 1] + + # step over filename + i += 1 + else: + usageCorrect = False + else: + usageCorrect = False + + elif sys.argv[i] not in ["-test", "-start"]: + settingsFile = sys.argv[i] + # usageCorrect = False + + i += 1 + + if "-test" in sys.argv and usageCorrect : + # run non-visually in test mode + # print "Testing " + settingsFile + try: + display = False + application = ebi(0) + if SuppressVersion: + Version = "" + application.main.createEngine(Version, Copyright, License, EBIDownloadPage, propertiesFile) + application.main.setDebug(Debug) + if settingsFile != "": + if application.main.loadSettings(settingsFile): + application.main.startImport(testOnly = True) + except: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + errorString = "ERROR: " + str(exceptionValue) + if Debug: + errorString += ': ' + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + alert(errorString) + + elif "-start" in sys.argv and usageCorrect : + try: + # run non-visually + # print "Running " + settingsFile + display = False + application = ebi(0) + if SuppressVersion: + Version = "" + application.main.createEngine(Version, Copyright, License, EBIDownloadPage, propertiesFile) + application.main.setDebug(Debug) + if settingsFile != "": + if application.main.loadSettings(settingsFile): + application.main.startImport(testOnly = False) + except: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + errorString = "ERROR: " + str(exceptionValue) + if Debug: + errorString += ': ' + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + alert(errorString) + + elif usageCorrect: + try: + # run visually starting with the main form + application = ebi(0) + if SuppressVersion: + Version = "" + application.main.createEngine(Version, Copyright, License, EBIDownloadPage, propertiesFile) + application.main.setDebug(Debug) + if settingsFile != "": + application.main.loadSettings(settingsFile) + elif "Configuration" in config.sections() and config.getboolean('Configuration','loadlastsettingsfile'): + try: + application.main.loadSettings(config.get('State','settingsfile')) + except: + if Debug: + alert(str(sys.exc_info()[1])) + + application.MainLoop() + except: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + errorString = "ERROR: " + str(exceptionValue) + if Debug: + errorString += ': ' + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + alert(errorString) + else: + alert(usageSyntax) + except: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + errorString = "ERROR: " + str(exceptionValue) + if "Errno 30" in errorString: + errorString += "\n\nEBI cannot read and write files in its current location. Try installing EBI in a different location." + if platform.system() == "Darwin": + errorString += "\n\nIf launching EBI from a mounted disk image (*.dmg) first copy the EBI package to Applications or another local directory." + if Debug: + errorString += "\n\n" + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + alert(errorString) + + +if __name__ == '__main__': + main() diff --git a/package-mac/source/ebi/ebi_manual.docx b/package-mac/source/ebi/ebi_manual.docx new file mode 100644 index 0000000..502b702 Binary files /dev/null and b/package-mac/source/ebi/ebi_manual.docx differ diff --git a/package-mac/source/ebi/ebibig.ico b/package-mac/source/ebi/ebibig.ico new file mode 100644 index 0000000..9051f20 Binary files /dev/null and b/package-mac/source/ebi/ebibig.ico differ diff --git a/package-mac/source/ebi/ebiicon.jpg b/package-mac/source/ebi/ebiicon.jpg new file mode 100644 index 0000000..df8b401 Binary files /dev/null and b/package-mac/source/ebi/ebiicon.jpg differ diff --git a/package-mac/source/ebi/ebismall.ico b/package-mac/source/ebi/ebismall.ico new file mode 100644 index 0000000..73edd33 Binary files /dev/null and b/package-mac/source/ebi/ebismall.ico differ diff --git a/package-mac/source/ebi/equellaclient41.py b/package-mac/source/ebi/equellaclient41.py new file mode 100644 index 0000000..3302fb4 --- /dev/null +++ b/package-mac/source/ebi/equellaclient41.py @@ -0,0 +1,1767 @@ +# equellaclient41.py +# +# Author: Adam Eijdenberg, Dytech Solutions +# Date: 2005 +# +# The EQUELLA SOAP abstraction layer for the EQUELLA Bulk Importer. Effectively a wrapper +# for EQUELLA's SOAP API for invocation by Engine.py. Includes the necessary HTTP network +# plumbing. To support EQUELLA down to version 4.1 it uses the EQUELLA 4.1 API and endpoint +# for the majority of functionality. Some functions included for features supported only in +# higher versions of EQUELLA (e.g. control of item ownership for 5.1+). +# +#~ MF - 2007 - added encoding parameter to AddFile method and created toXml and printXml for MockClient +#~ MF - 2008 - fixed bug in search for multiple itemdefs +#~ JK - 2009 - removeNode() removes all matching nodes instead of just the first +#~ JK - 2010 - added support for multiple cookies (needed for clustering) +#~ JK - 2010 - improved file chunking for large file attachments e.g. 1 GB and greater +#~ JK - 2011 - replaced HTTPConnection and HTTPSConnection with urllib2 +#~ JK - 2012 - added Engine.py-dependent logging +#~ JK - 2012 - replaced custom cookie management code with standard Python cookielib library +#~ JK - 2012 - added additional HTTP headers +#~ JK - 2013 - improved xpath indexes support +#~ JK - 2013 - removed the ability to use createNode with complex Python datatypes +#~ JK - 2013 - added getText() and getFile() functions to allow download of attachments in the session +#~ JK - 2013 - removeNode() also supports removing attributes +#~ JK - 2014 - removeNode() supports xpath indexes +#~ JK - 2015 - added support for additional xpath predicates + +import time, md5, urllib +import sys, urllib2, re, cookielib +from xml.dom.minidom import parse, parseString +from xml.dom import Node +from binascii import b2a_base64 +from urlparse import urlparse +import codecs +import os, os.path, traceback, time +from string import ascii_letters +import wx +import ssl + +ASCII_ENC = codecs.getencoder('us-ascii') + +SOAP_HEADER_TOKEN = '%(token)s' + +SOAP_REQUEST = '%(header)s%(params)s' + +SOAP_PARAMETER = '%(value)s' + +STANDARD_INTERFACE = {'url':'services/SoapService51', 'namespace':'http://soap.remoting.web.tle.com'} + +HTTP_HEADERS = { +'Content-type': 'text/xml', +'SOAPAction': '', +'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6', +'Accept-Language':'en-us,en;q=0.5', +'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.7', +'Keep-Alive':'300' +} + +SOAP_ENDPOINT_V1 = 'services/SoapInterfaceV1' +SOAP_ENDPOINT_V2 = 'services/SoapInterfaceV2' +SOAP_ENDPOINT_V41 = 'services/SoapService41' +SOAP_ENDPOINT_V51 = 'services/SoapService51' + +def escape(s): + return s.replace ('&', '&').replace ('<', '<').replace ('>', '>').replace ('"', '"').replace ("'", ''') + +def value_as_string (node): + node.normalize () + return ''.join ([x.nodeValue for x in node.childNodes]) + +def get_named_child_value (node, name): + return value_as_string(node.getElementsByTagName(name)[0]) + +def stripNode(node, recurse=False): + nodesToRemove = [] + nodeToBeStripped = False + for childNode in node.childNodes: + # list empty text nodes (to remove if any should be) + if (childNode.nodeType == Node.TEXT_NODE and childNode.nodeValue.strip() == ""): + nodesToRemove.append(childNode) + + # only remove empty text nodes if not a leaf node (i.e. a child element exists) + if childNode.nodeType == Node.ELEMENT_NODE: + nodeToBeStripped = True + + # remove flagged text nodes + if nodeToBeStripped: + for childNode in nodesToRemove: + node.removeChild(childNode) + + # recurse if specified + if recurse: + for childNode in node.childNodes: + if childNode.nodeType == Node.ELEMENT_NODE: + stripNode(childNode, True) + +def clean_unicode (s): + if s.__class__ == unicode: + return ASCII_ENC (s, 'xmlcharrefreplace') [0] + else: + return s + +def value_as_node_or_string (cur): + cur.normalize () + if len (cur.childNodes) == 1: + if cur.firstChild.nodeType == cur.TEXT_NODE: + return value_as_string (cur) + # Empty node + return '' + +def generate_soap_envelope (name, params, ns, token=None): + # Need to handle arrays + def p(value): + if isinstance(value, list): + buf = '' + for i in value: + buf += ''+ escape (clean_unicode (i)) + '' + return buf + else: + return escape (clean_unicode (value)) + def arrayType(value, type): + if isinstance(value, list): + return ' ns1:arrayType="%s[%s]"' % (type, len(value)) + else: + return '' + def t(value, type): + if isinstance(value, list): + return 'ns1:Array' + else: + return type + + return SOAP_REQUEST % { + 'ns': ns, + 'method': name, + 'params': '' if len(params) == 0 else ''.join([SOAP_PARAMETER % { + 'name': 'in' + str(i), + 'type': t(v[2], v[1]), + 'value': p(v[2]), + 'arrayType': arrayType(v[2], v[1]) + } for i, v in enumerate(params)]), + 'header': '' if token == None or len(token) == 0 else SOAP_HEADER_TOKEN % {'token': token} + } + +def urlEncode(text): + return urllib.urlencode ({'q': text}) [2:] + +def generateToken(username, sharedSecretId, sharedSecretValue): + seed = str (int (time.time ())) + '000' + id2 = urlEncode (sharedSecretId) + if(not(sharedSecretId == '')): + id2 += ':' + + return '%s:%s%s:%s' % ( + urlEncode(username), + id2, + seed, + binascii.b2a_base64(hashlib.md5( + username + sharedSecretId + seed + sharedSecretValue + ).digest()) + ) + +# Class designed to make communicated with TLE very easy! +class TLEClient: + # First, instantiate an instance of this class. + # e,g, client = TLEClient ('lcms.yourinstitution.edu.au', 'admin', 'youradminpasssword') + def __init__ (self, owner, institutionUrl, username, password, proxy = "", proxyusername = "", proxypassword = "", debug = False, sso=0): + + self.debug = debug + self.owner = owner + + # trim off logon.do if it is in url + self.institutionUrl = institutionUrl + urlLogonPagePos = self.institutionUrl.find("/logon.do") + if urlLogonPagePos != -1: + self.institutionUrl = self.institutionUrl[:urlLogonPagePos] + + # make certain instituion URL does not end with a slash + if self.institutionUrl.endswith("/"): + self.institutionUrl = self.institutionUrl[:-1] + + self.protocol = urlparse(self.institutionUrl)[0] + self.host = urlparse(self.institutionUrl)[1] + self.context = urlparse(self.institutionUrl)[2] + + # cookie management + #self._cookieJar = cookielib.CookieJar() + #self._cookieProcessor = urllib2.HTTPCookieProcessor(self._cookieJar) + self._cookieJar = [] + + # set proxy + self.proxy = proxy + self.proxyusername = proxyusername + self.proxypassword = proxypassword + if self.proxy != "": + password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() + password_mgr.add_password(None, self.proxy, self.proxyusername, self.proxypassword) + proxy_auth_handler = urllib2.ProxyBasicAuthHandler(password_mgr) + proxy_handler = urllib2.ProxyHandler({"http": self.proxy}) + + # build URL opener with proxy + #opener = urllib2.build_opener(proxy_handler, proxy_auth_handler, self._cookieProcessor) + opener = urllib2.build_opener(proxy_handler, proxy_auth_handler) + else: + # build URL opener without proxy + #opener = urllib2.build_opener(self._cookieProcessor) + opener = urllib2.build_opener() + urllib2.install_opener(opener) + + if sso: + self.sessionid = self._createSoapSessionFromToken (createSSOToken (username, password)) + else: + self.sessionid = self._createSoapSession (username, password) + + def _call (self, name, args, returns=1, facade=SOAP_ENDPOINT_V41, ns='http://soap.remoting.web.tle.com'): + try: + headers = {} + headers.update(HTTP_HEADERS) + if len(self._cookieJar) > 0: + headers['Cookie'] = "; ".join(self._cookieJar) + + endpointUrl = self.institutionUrl + "/" + facade + wsenvelope = generate_soap_envelope (name, args, ns) + + if self.owner.networkLogging: + self.owner.echo("\n\n*************************************************************") + self.owner.echo("---------------- COMMUNICATION WITH EQUELLA -----------------") + self.owner.echo("-------------------------------------------------------------") + self.owner.echo("SOAP METHOD: %s()\n" % name) + self.owner.echo("HTTP REQUEST:\n") + self.owner.echo(" Endpoint:\n%s\n" % endpointUrl) + self.owner.echo(" Headers:\n%s\n" % headers) + if name not in ['uploadFile'] or len(str(args)) < 1000: + if name not in ['login']: + self.owner.echo(" Request Body:\n" + wsenvelope + "\n") + self.owner.echo(" SOAP Input Parameters:\n" + str(args) + "\n\n") + else: + # generate a copy of the args and SOAP message with password masked + maskedargs = (args[0], (args[1][0], args[1][1], "??????")) + wsmaskedenvelope = generate_soap_envelope (name, maskedargs, ns) + self.owner.echo(" Request Body:\n" + wsmaskedenvelope + "\n") + self.owner.echo(" SOAP Input Parameters:\n" + str(maskedargs) + "\n\n") + else: + self.owner.echo(" Request Body:\n<%s characters including base64 data>\n" % len(wsenvelope)) + printableArgs = "(%s, %s, ('%s', '%s', <%s characters of base64 data>), %s)" % (args[0], args[1], args[2][0], args[2][1], len(args[2][2]), args[3]) + self.owner.echo(" SOAP Input Parameters:\n" + printableArgs + "\n\n") + self.owner.echo(" Cookies:\n%s\n" % self._cookieJar) + + # make request + request = urllib2.Request(endpointUrl, wsenvelope, headers) + context = ssl._create_unverified_context() + response = urllib2.urlopen(request, context=context) + #response = urllib2.urlopen(request) + + # read response and close connection + s = response.read () + response.close() + + responseInfo = response.info() + headers = {} + cookie = responseInfo.getheader('set-cookie') + if cookie is not None: + for cookie_part in cookie.split(','): + for cookie_name in cookie_part.split(','): + if not cookie_name.upper().split("=")[0].strip() in ["PATH", "DOMAIN", "EXPIRES", "SECURE", + "HTTPONLY"]: + # save cookie + self._cookieJar.append(cookie_name) + + if self.owner.networkLogging: + self.owner.echo("HTTP RESPONSE:\n") + self.owner.echo(" Headers:\n%s\n" % response.info().headers) + self.owner.echo(" Cookies:\n%s\n" % self._cookieJar) + self.owner.echo(" Response Body:\n%s\n" % s) + + except urllib2.HTTPError, e: + httpErrorBody = "" + httpError = e.reason + try: + httpErrorBody = e.read() + except: + pass + if self.owner.networkLogging: + self.owner.echo("HTTP ERROR CODE: " + str(e.code)) + self.owner.echo(" Error Reason:\n%s\n" % httpError) + if httpErrorBody != "": + self.owner.echo(" Response Body:\n%s\n" % httpErrorBody) + self.owner.echo("-------------------------------------------------------------") + self.owner.echo("--------------------- END COMMUNICATION ---------------------") + self.owner.echo("*************************************************************\n\n") + + errorString = "" + + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + errorString += "\n" + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + "\n" + + if httpErrorBody != "": + try: + errordom = parseString(httpErrorBody) + faultstring = errordom.firstChild.getElementsByTagName("soap:Body")[0].getElementsByTagName("soap:Fault")[0].getElementsByTagName("faultstring")[0].firstChild.nodeValue + errorString += faultstring + except: + errorString += httpError + else: + errorString += httpError + + raise Exception, errorString + +## except urllib2.URLError, e: +## raise Exception, str(e.args[0][1]) + + except: + if self.owner.networkLogging: + self.owner.echo("ERROR: " + str(sys.exc_info()[1])) + self.owner.echo("-------------------------------------------------------------") + self.owner.echo("--------------------- END COMMUNICATION ---------------------") + self.owner.echo("*************************************************************\n\n") + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + errorString = "\n" + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + "\n" + else: + errorString = sys.exc_info()[1] + raise Exception, errorString + + try: + dom = parseString(s) + except: + if self.owner.networkLogging: + self.owner.echo("ERROR: " + str(sys.exc_info()[1])) + self.owner.echo("-------------------------------------------------------------") + self.owner.echo("--------------------- END COMMUNICATION ---------------------") + self.owner.echo("*************************************************************\n\n") + errorString = "" + if self.debug: + exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() + errorString += "\n" + ''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)) + "\n" + errorString += "Cannot parse server response as XML\n" + s + raise Exception, errorString + + if len (dom.getElementsByTagNameNS ('http://schemas.xmlsoap.org/soap/envelope/', 'Fault')): + raise Exception, 'Server returned following SOAP error: %s' % dom.toprettyxml () + elif returns: # then return a result + + returnValue = dom.firstChild.getElementsByTagName("soap:Body")[0].firstChild.firstChild + + if self.owner.networkLogging: + returnValueString = "n/a" + if returnValue != None: + try: + returnValueString = value_as_string(returnValue) + except: + returnValueString = str(returnValue) + + self.owner.echo(" SOAP Return Parameter:\n" + str(returnValueString)) + self.owner.echo("-------------------------------------------------------------") + self.owner.echo("--------------------- END COMMUNICATION ---------------------") + self.owner.echo("*************************************************************\n\n") + + + return returnValue + + if self.owner.networkLogging: + self.owner.echo(" SOAP Return Parameter:\nn/a") + self.owner.echo("-------------------------------------------------------------") + self.owner.echo("--------------------- END COMMUNICATION ---------------------") + self.owner.echo("*************************************************************\n\n") + + + def _createSoapSessionFromToken (self, token): + result = self._call ('loginWithToken', ( + ('token', 'xsd:string', token), + )) + return value_as_string (result) + + def _createSoapSession (self, username, password): + result = self._call ('login', ( + ('username', 'xsd:string', username), + ('password', 'xsd:string', password), + )) + return value_as_string (result) + + def logout (self): + self._call ('logout', ( + )) + + def getFile(self, url, filepath): + headers = {} +## headers.update(HTTP_HEADERS) + headers.update(HTTP_HEADERS) + if len(self._cookieJar) > 0: + headers['Cookie'] = "; ".join(self._cookieJar) + + retry = True + while retry: + try: + request = urllib2.Request(url, headers=headers) + context = ssl._create_unverified_context() + response = urllib2.urlopen(request, context=context) + f = open(filepath, "wb") + f.write(response.read()) + try: + f.close() + except: + pass + retry = False + + except urllib2.URLError, err: + if hasattr(err, 'reason'): + if str(err.reason).find("10054") != -1: + print err.reason + print "Retrying..." + else: + retry = False + raise Exception, "url/http error: " + str(err.reason) + if hasattr(err, 'code'): + retry = False + raise Exception, "url/http error code: " + str(err.code) + except: + err = sys.exc_info()[1] + if str(err).find("10054") != -1: + print err + print "Retrying..." + else: + retry = False + raise err + + return response.info() + + def getText(self, url): + headers = {} + headers.update(HTTP_HEADERS) + retry = True + while retry: + try: + request = urllib2.Request(url, headers=headers) + context = ssl._create_unverified_context() + response = urllib2.urlopen(request, context=context) + retry = False + return response.read() + except urllib2.URLError, err: + if str(err.reason).find("10054"): + print err.reason + print "Retrying..." + else: + retry = False + raise err.reason + except: + err = sys.exc_info()[1] + if str(err).find("10054") != -1: + print err + print "Retrying..." + else: + retry = False + raise err + + return response.info(), data + + def _unzipFile (self, stagingid, zipfile, outpath): + self._call ('unzipFile', ( + ('item_uuid', 'xsd:string', stagingid), + ('zipfile', 'xsd:string', zipfile), + ('outpath', 'xsd:string', outpath), + ), returns=0) + + def _enumerateItemDefs (self): + result = self._call('getContributableCollections', ( + )) + return dict ([(get_named_child_value (itemdef, 'name'), {'uuid': get_named_child_value (itemdef, 'uuid')}) for itemdef in parseString (value_as_string (result)).getElementsByTagName ('itemdef')]) + + def _newItem (self, itemdefid): + result = self._call ('newItem', ( + ('itemdefid', 'xsd:string', itemdefid), + )) + return parseString (value_as_string (result)) + + # _newVersionItem() only supported in 4.1 and higher + def _newVersionItem (self, itemid, itemversion, copyattachments): + result = self._call ('newVersionItem', ( + ('itemid', 'xsd:string', itemid), + ('version', 'xsd:int', itemversion), + ('copyattachments', 'xsd:boolean', str(copyattachments)), + ), 1, SOAP_ENDPOINT_V41) + return parseString (value_as_string (result)) + + def _startEdit (self, itemid, itemversion, copyattachments): + result = self._call ('editItem', ( + ('itemid', 'xsd:string', itemid), + ('version', 'xsd:int', str (itemversion)), + ('copyattachments', 'xsd:boolean', str (copyattachments)), + )) + return parseString (value_as_string (result)) + + def _forceUnlock (self, itemid, itemversion): + result = self._call ('unlock', ( + ('itemid', 'xsd:string', itemid), + ('version', 'xsd:int', str (itemversion)), + ), returns=0) + return result + + def _stopEdit (self, xml, submit): + result = self._call ('saveItem', ( + ('itemXML', 'xsd:string', xml), + ('bSubmit', 'xsd:boolean', submit), + )) + return result + + def _cancelEdit (self, itemid, itemversion): + result = self._call ('cancelItemEdit', ( + ('itemid', 'xsd:string', itemid), + ('version', 'xsd:int', str (itemversion)), + )) + return result + + def _uploadFile (self, stagingid, filename, data, overwrite): + result = self._call ('uploadFile', ( + ('item_uuid', 'xsd:string', stagingid), + ('filename', 'xsd:string', filename), + ('data', 'xsd:string', data), + ('overwrite', 'xsd:boolean', overwrite), + )) + return result + + def _deleteAttachmentFile (self, stagingid, filename): + result = self._call ('deleteFile', ( + ('item_uuid', 'xsd:string', stagingid), + ('filename', 'xsd:string', filename), + )) + return result + + def _deleteItem (self, itemid, itemversion): + result = self._call ('deleteItem', ( + ('itemid', 'xsd:string', itemid), + ('version', 'xsd:int', str (itemversion)), + )) + return result + + def getItem (self, itemid, itemversion, select=''): + result = self._call ('getItem', ( + ('itemid', 'xsd:string', itemid), + ('version', 'xsd:int', str (itemversion)), + ('select', 'xsd:string', select), + )) + return PropBagEx(value_as_string(result)) + + def getItemFilenames (self, itemUuid, itemVersion, path, system): + paramSystem = 'false' + if system: + paramSystem = 'true' + result = self._call ('getItemFilenames', ( + ('itemUuid', 'xsd:string', itemUuid), + ('itemVersion', 'xsd:int', str (itemVersion)), + ('path', 'xsd:string', path), + ('system', 'xsd:boolean', paramSystem), + ), 1, SOAP_ENDPOINT_V51) + + resultList = [] + for childNode in result.childNodes: + resultList.append(childNode.firstChild.nodeValue) + return resultList + + + def queryCount (self, itemdefs, where): + result = self._call ('queryCount', ( + ('itemdefs', 'xsd:string', itemdefs), + ('where', 'xsd:string', where), + )) + return int(value_as_string(result)) + + # Return an itemdef UUID given a human displayable name. + # e.g. itemdefUUID = client.getItemdefUUID ('K-12 Educational Resource') + def getItemdefUUID (self, itemdefName): + return self._enumerateItemDefs () [itemdefName] ['uuid'] + + # Return an ident given a human displayable name. + # e.g. itemdefUUID = client.getItemdefUUID ('K-12 Educational Resource') + def getItemdefIdent (self, itemdefName): + return self._enumerateItemDefs () [itemdefName] ['ident'] + + # Create a new repository item of the type specified. See NewItemClient for methods that can be called on the return type. + # e.g. item = client.createNewItem (itemdefUUID) + def createNewItem (self, itemdefid): + rv = NewItemClient(self, self.owner, self._newItem(itemdefid), debug=self.debug) + return rv + + # Create a new version of the item specified. See NewItemClient for methods that can be called on the return type. + # e.g. item = client.newVersionItem(itemUUID, "2", itemdefUUID) + # NOTE: only supported in 4.1 and higher + def newVersionItem(self, itemid, version, copyattachments = True): + rv = NewItemClient(self, self.owner, self._newVersionItem(itemid, str(version), copyattachments), debug=self.debug) + return rv + + # Edit particular item. See NewItemClient for methods that can be called on the return type. + # e.g. item = client.createNewItem (itemdefUUID) + def editItem (self, itemid, version, copyattachments): + dom = self._startEdit (itemid, version, copyattachments) + if not len (dom.getElementsByTagName ('item')): + self._forceUnlock (itemid, version) + dom = self._startEdit (itemid, version, copyattachments) + rv = NewItemClient (self, self.owner, dom, newversion=0, copyattachments=(copyattachments == 'true'), debug=self.debug) + return rv + + def search(self, offset=0, limit=10, select='*', itemdefs=[], where='', query='', onlyLive=True, orderType=0, reverseOrder=False): + paramReverseOrder = 'false' + if reverseOrder: + paramReverseOrder = 'true' + + paramOnlyLive = 'false' + if onlyLive: + paramOnlyLive = 'true' + + result = self._call ('searchItems', ( + ('freetext', 'xsd:string', query), + ('collectionUuids', 'xsd:string', itemdefs), + ('whereClause', 'xsd:string', where), + ('onlyLive', 'xsd:boolean', paramOnlyLive), + ('orderType', 'xsd:int', str (orderType)), + ('reverseOrder', 'xsd:boolean', paramReverseOrder), + ('offset', 'xsd:int', str(offset)), + ('limit', 'xsd:int', str(limit)), + )) + return PropBagEx(value_as_string(result)) + + def searchItemsFast(self, freetext='', collectionUuids=[], whereClause='', onlyLive=True, orderType=0, reverseOrder=False, offset=0, length=50, resultCategories=["basic"]): + paramReverseOrder = 'false' + if reverseOrder: + paramReverseOrder = 'true' + + paramOnlyLive = 'false' + if onlyLive: + paramOnlyLive = 'true' + + result = self._call ('searchItemsFast', ( + ('freetext', 'xsd:string', freetext), + ('collectionUuids', 'xsd:string', collectionUuids), + ('whereClause', 'xsd:string', whereClause), + ('onlyLive', 'xsd:boolean', paramOnlyLive), + ('orderType', 'xsd:int', str (orderType)), + ('reverseOrder', 'xsd:boolean', paramReverseOrder), + ('offset', 'xsd:int', str(offset)), + ('length', 'xsd:int', str(length)), + ('resultCategories', 'xsd:string', resultCategories), + )) + return PropBagEx(value_as_string(result)) + + def setOwner (self, itemid, itemversion, ownerid): + result = self._call ('setOwner', ( + ('itemid', 'xsd:string', itemid), + ('version', 'xsd:int', str (itemversion)), + ('userId', 'xsd:string', ownerid), + ), 1, facade = SOAP_ENDPOINT_V51) + + def setOwnerByUsername(self, itemID, version, username, saveNonexistentUsernamesAsIDs = False): + matchingUsers = self.searchUsersByGroup("", username) + matchingUserNodes = matchingUsers.getNodes("user", False) + + # if any matches get first matching user + if len(matchingUserNodes) > 0: + userID = matchingUserNodes[0].getElementsByTagName("uuid")[0].firstChild.nodeValue + + # set item owner + self.setOwner(itemID, version, userID) + else: + if saveNonexistentUsernamesAsIDs: + self.setOwner(itemID, version, username) + else: + raise Exception, "User [%s] not found in EQUELLA" % username + + def addSharedOwner (self, itemid, itemversion, ownerid): + result = self._call ('addSharedOwner', ( + ('itemid', 'xsd:string', itemid), + ('version', 'xsd:int', str (itemversion)), + ('userId', 'xsd:string', ownerid), + )) + + def addSharedOwners (self, itemid, itemversion, collaboratorUsernames, saveNonexistentUsernamesAsIDs = False): + for username in collaboratorUsernames: + + matchingUsers = self.searchUsersByGroup("", username) + matchingUserNodes = matchingUsers.getNodes("user", False) + + # if any matches get first matching user + if len(matchingUserNodes) > 0: + userid = matchingUserNodes[0].getElementsByTagName("uuid")[0].firstChild.nodeValue + + # set item owner + self.addSharedOwner(itemid, itemversion, userid) + else: + if saveNonexistentUsernamesAsIDs: + self.addSharedOwner(itemid, itemversion, username) + else: + raise Exception, "User [%s] not found in EQUELLA" % username + + def removeSharedOwner (self, itemid, itemversion, ownerid): + result = self._call ('removeSharedOwner', ( + ('itemid', 'xsd:string', itemid), + ('version', 'xsd:int', str (itemversion)), + ('userId', 'xsd:string', ownerid), + )) + + + def getUser (self, userId): + result = self._call ('getUser', ( + ('userId', 'xsd:string', userId), + ), 1, facade = SOAP_ENDPOINT_V51) + return PropBagEx(value_as_string(result)) + + def searchUsersByGroup (self, groupUuid, searchString): + result = self._call ('searchUsersByGroup', ( + ('groupUuid', 'xsd:string', groupUuid), + ('searchString', 'xsd:string', searchString), + ), 1, SOAP_ENDPOINT_V51) + return PropBagEx(value_as_string(result)) + + def activateItemAttachments (self, uuid, version, courseCode, attachments): + self._call ('activateItemAttachments', ( + ('uuid', 'xsd:string', uuid), + ('version', 'xsd:int', version), + ('courseCode', 'xsd:string', courseCode), + ('attachments', 'xsd:string', attachments), + )) + + def addUser (self, uuid, username, password, firstname, lastname, email): + return value_as_string(self._call ('addUser', ( + ('ssid', 'xsd:string', self.sessionid), + ('uuid', 'xsd:string', uuid), + ('name', 'xsd:string', username), + ('password', 'xsd:string', password), + ('first', 'xsd:string', firstname), + ('last', 'xsd:string', lastname), + ('email', 'xsd:string', email), + ), facade=SOAP_INTERFACE_V2)) + + def editUser (self, uuid, username, password, firstname, lastname, email): + return value_as_string(self._call ('editUser', ( + ('ssid', 'xsd:string', self.sessionid), + ('uuid', 'xsd:string', uuid), + ('name', 'xsd:string', username), + ('password', 'xsd:string', password), + ('first', 'xsd:string', firstname), + ('last', 'xsd:string', lastname), + ('email', 'xsd:string', email), + ), facade=SOAP_INTERFACE_V2)) + + def removeUser (self, uuid): + self._call ('removeUser', ( + ('ssid', 'xsd:string', self.sessionid), + ('uuid', 'xsd:string', uuid), + ), facade=SOAP_INTERFACE_V2) + + def addUserToGroup (self, uuid, groupId): + self._call ('addUserToGroup', ( + ('ssid', 'xsd:string', self.sessionid), + ('uuid', 'xsd:string', uuid), + ('groupid', 'xsd:string', groupId), + ), facade=SOAP_INTERFACE_V2) + + def removeUserFromGroup (self, uuid, groupId): + self._call ('removeUserFromGroup', ( + ('ssid', 'xsd:string', self.sessionid), + ('uuid', 'xsd:string', uuid), + ('groupid', 'xsd:string', groupId), + ), facade=SOAP_INTERFACE_V2) + + def removeUserFromAllGroups (self, userId): + self._call ('removeUserFromAllGroups', ( + ('ssid', 'xsd:string', self.sessionid), + ('userUuid', 'xsd:string', userId), + ), facade=SOAP_INTERFACE_V2) + + def isUserInGroup (self, userId, groupId): + return value_as_string(self._call ('isUserInGroup', ( + ('ssid', 'xsd:string', self.sessionid), + ('userUuid', 'xsd:string', userId), + ('groupUuid', 'xsd:string', groupId), + ), facade=SOAP_INTERFACE_V2)) == 'true' + + +class NewItemClient: + def __init__ (self, parClient, owner, newDom, newversion=0, copyattachments=1, debug = False): + self.debug = debug + self.owner = owner + self.parClient = parClient + self.newDom = newDom + self.prop = PropBagEx(self.newDom.firstChild) + self.xml = self.newDom.firstChild + self.uuid = self.prop.getNode("item/@id") + self.version = self.prop.getNode("item/@version") + + if copyattachments: + self.stagingid = self.prop.getNode("item/staging") + + # remove old version references to non-existent start-pages + if newversion and not copyattachments: + attachmentsNode.removeNode("item/attachments/attachment") + + def getUUID (self): + return self.uuid + + def getVersion (self): + return self.version + + def getItemdefUUID (self): + return self.prop.getNode("item/@itemdefid") + + def read_in_chunks(self, file_object, chunk_size=(1024 * 1024)): + while True: + data = file_object.read(chunk_size) + if not data: + break + yield data + + # Upload a file as an attachment to this item. path is where the item will live inside of the repository, and should not contain a preceding slash. + # e.g. item.attachFile ('support/song.wav', file ('c:\\Documents and Settings\\adame\\Desktop\\song.wav', 'rb')) + # Parent directories are automatically created as required. + # Uploads file in chunks of 16MB. If file is large (e.g. over 16MB) pass in parameter showstatus as a prefix to a progress report + # e.g. item.attachFile ('support/song.wav', file ('video.avi', 'rb'), ' Progress: ') + def attachFile (self, path, attachment, showstatus=None, chunk_size=(1024 * 2048)): + if showstatus: + if self.debug: + self.owner.echo(showstatus + " Uploading...") + else: + self.owner.log.SetReadOnly(False) + self.owner.log.AppendText(showstatus + " Uploading...") + self.owner.log.SetReadOnly(True) + sys.stdout.write(showstatus + " Uploading...") + sys.stdout.flush() + try: + firstChunk = "true" + filesize = os.path.getsize(attachment.name) + uploaded = 0 + for chunk in self.read_in_chunks(attachment, chunk_size): + wx.GetApp().Yield() + if self.owner.StopProcessing: + if self.debug: + self.owner.echo(showstatus + " Halted by user") + else: + self.owner.log.SetReadOnly(False) + self.owner.log.AppendText("\n") + self.owner.log.SetReadOnly(True) + + sys.stdout.write("Halted by user\n") + self.owner.echo(showstatus + " Uploading...Halted by user", False) + break + uploaded += len(chunk) + encodedChunk = b2a_base64(chunk) + self.parClient._uploadFile (self.stagingid, path, encodedChunk, firstChunk) + + if firstChunk == "true": + firstChunk = "false" + if showstatus: + if self.debug: + progressReport = showstatus + " chunk=%s, uploaded=%s/%s" % (len(chunk), uploaded, filesize) + self.owner.echo(progressReport) + if uploaded >= filesize: + self.owner.echo(" Done") + self.owner.tryPausing(" [Paused]") + else: + if uploaded >= filesize: + self.owner.echo(showstatus + " Uploading...Done", False) + sys.stdout.write("Done\n") + + self.owner.log.DocumentEnd() + self.owner.log.SetReadOnly(False) + self.owner.log.DelLineLeft() + self.owner.log.AppendText(showstatus + " Uploading...Done\n") + self.owner.log.SetReadOnly(True) + else: + sys.stdout.write(".") + sys.stdout.flush() + + progressString = showstatus + " Uploading...%s%%" % ((uploaded * 100)/ filesize) + self.owner.log.DocumentEnd() + self.owner.log.SetReadOnly(False) + self.owner.log.DelLineLeft() + self.owner.log.AppendText(progressString) + self.owner.log.SetReadOnly(True) + self.owner.tryPausing(" [Paused]", newline = True) + except: + if not self.debug: + sys.stdout.write("\n") + raise + + def unzipFile (self, path, name): + self.parClient._unzipFile (self.stagingid, path, name) + + # Uploads an IMS package + def attachIMS (self, file, filename='package.zip', title='', showstatus=None, upload = True, size=1024, uuid="", chunk_size=(4048 * 4048)): + imsfilename = '_IMS/' + filename + if upload: + self.attachFile (imsfilename, file, showstatus, chunk_size) + if not self.owner.StopProcessing: + self.parClient._unzipFile (self.stagingid, imsfilename, filename) + + self.prop.setNode("item/itembody/packagefile", filename) + self.prop.setNode("item/itembody/packagefile/@name", title) + self.prop.setNode("item/itembody/packagefile/@size", str(size)) + self.prop.setNode("item/itembody/packagefile/@stored", "true") + if uuid != '': + self.prop.setNode("item/itembody/packagefile/@uuid", uuid) + + + # Uploads a SCORM package + def attachSCORM (self, file, filename, description, showstatus=None, upload = True, size=1024, uuid = '', chunk_size=(4048 * 4048)): + if upload: + scormfilename = '_SCORM/' + filename + self.attachFile(scormfilename, file, showstatus, chunk_size) + self.parClient._unzipFile (self.stagingid, scormfilename, filename) + + attachment = self.prop.newSubtree("item/attachments/attachment") + attachment.createNode("@type", "custom") + attachment.createNode("type", "scorm") + attachment.createNode("attributes/entry/string", "fileSize") + attachment.createNode("attributes/entry/long", str(size)) + scormVersionEntry = attachment.newSubtree("attributes/entry") + scormVersionEntry.createNode("string", "SCORM_VERSION") + scormVersionEntry.createNode("string", "1.2") + attachment.createNode("file", filename) + attachment.createNode("description", description) + if uuid != '': + attachment.createNode("uuid", uuid) + + def attachResource (self, resourceItemUuid, resourceItemVersion, resourceDescription, uuid = '', attachmentUuid = ""): + # create attachment subtree + attachment = self.prop.newSubtree("item/attachments/attachment") + attachment.createNode("@type", "custom") + attachment.createNode("type", "resource") + if attachmentUuid != "": + attachment.createNode("file", attachmentUuid) + else: + attachment.createNode("file", "") + attachment.createNode("description", resourceDescription) + if uuid != '': + attachment.createNode("uuid", uuid) + + # create uuid entry and append to attributes + attributeEntry = attachment.newSubtree("attributes/entry") + attributeEntry.createNode("string", "uuid") + attributeEntry.createNode("string", resourceItemUuid) + + # create type entry and append to attributes + attributeEntry = attachment.newSubtree("attributes/entry") + attributeEntry.createNode("string", "type") + if attachmentUuid == "": + attributeEntry.createNode("string", "p") + else: + attributeEntry.createNode("string", "a") + + # create version entry and append to attributes + attributeEntry = attachment.newSubtree("attributes/entry") + attributeEntry.createNode("string", "version") + attributeEntry.createNode("int", str(resourceItemVersion)) + + # Mark an attached file as a start page to appear on the item summary page. + # e.g. item.addStartPage ('Great song!', 'support/song.wav') + def addStartPage (self, description, path, size=1024, uuid='', thumbnail = ""): + + # delete existing attachment noes of the same /file and /description + self.prop.removeNode("item/attachments/attachment[file = '%s']" % path) + + attachment = self.prop.newSubtree("item/attachments/attachment") + attachment.createNode("@type", "local") + attachment.createNode("file", path) + attachment.createNode("description", description) + attachment.createNode("size", str(size)) + if uuid != '': + attachment.createNode("uuid", uuid) + if thumbnail != "": + attachment.createNode("thumbnail", thumbnail) + + def deleteAttachments(self): + self.getXml().removeNode('item/attachments/attachment') + self.parClient._deleteAttachmentFile(self.stagingid,"") + + # Add a URL as a resource to this item. + # e.g. item.addUrl ('Interesting link', 'http://www.thelearningedge.com.au/') + def addUrl (self, description, url, uuid=''): + attachment = self.prop.newSubtree("item/attachments/attachment") + attachment.createNode("@type", "remote") + attachment.createNode("conversion", "true") + attachment.createNode("file", url) + attachment.createNode("description", description) + if uuid != '': + attachment.createNode("uuid", uuid) + + # Print tabbed XML for this item, useful for debugging. + def printXml (self): + print ASCII_ENC (self.newDom.toprettyxml (), 'xmlcharrefreplace') [0] + + # Print tabbed XML for this item, useful for debugging. + def toXml (self, enc="utf-8"): + return self.newDom.toprettyxml (" ", "\n", enc) + + def forceUnlock(self): + self.parClient._forceUnlock(self.getUUID (), self.getVersion (), self.getItemdefUUID ()) + + def cancelEdit(self): + self.parClient._cancelEdit(self.getUUID (), self.getVersion (), self.getItemdefUUID ()) + + def delete (self): + self.parClient._deleteItem(self.getUUID (), self.getVersion (), self.getItemdefUUID ()) + + # Save this item into the repository. + # e.g. item.submit () + def submit (self, workflow=1): + self.parClient._stopEdit (self.newDom.toxml(), ('false', 'true') [workflow]) + + def getXml(self): + return self.prop + +# PropBag classes - was propbag.py + +class PropBagEx: + def __init__ (self, s, encoding="utf8"): + if isinstance (s, PropBagEx) : + self.document = s.document + self.root = s.root + elif isinstance (s, str) or isinstance(s, unicode): + self.document = parseString(s.encode(encoding)) + self.root = None + for childNode in self.document.childNodes: + if childNode.nodeType == Node.ELEMENT_NODE: + self.root = childNode + break + elif isinstance (s, file) : + self.document = parse (s) + self.root = None + for childNode in self.document.childNodes: + if childNode.nodeType == Node.ELEMENT_NODE: + self.root = childNode + break + else: + self.document = s.ownerDocument + self.root = s + self.xpath = XPath() + + def getNodes (self, xpath, string=True): + return self.xpath.selectNodes(xpath, self.root, string) + + # Get an XML node on this item. xpath should begin with item, but should not have a preceding slash. + # e.g. item.getNode ('item/description', 'This item describes ....') + def getNode (self, xpath): + nodes = self.getNodes (xpath) + if len(nodes) > 0: + return nodes[0] + return None + + def _createNewNodes(self, xpath, onlyOne = False): + # determine how much of xpath already exists + rhs = xpath + xpathParts = [] + i = 0 + missingParents = False + while rhs != "": + lhs, rhs, delimter = self.xpath.splitFirstOuter(rhs, ["/"]) + xpathParts.append(lhs) + if not missingParents and rhs != "" and len(self.getNodes("/".join(xpathParts))) != 0: + i += 1 + else: + missingParents = True + if i != 0: + parents = self.getNodes("/".join(xpathParts[0:i]), False) + else: + parents = self.getNodes("", False) + + # loop over last existing nodes in xpath and add necessary elements and attributes + # and return leaf nodes + returnNodes = [] + for parent in parents: + for j, childName in enumerate(xpathParts[i:]): + k = childName.find("[") + if k != -1: + childName = childName[:k] + if childName[0] != "@": + childNode = self.document.createElement(childName) + if j + 1 == len(xpathParts[i:]): + returnNodes.append(childNode) + parent.appendChild(childNode) + parent = childNode + elif j + 1 == len(xpathParts[i:]): + parent.setAttribute(childName[1:], "") + returnNodes.append(parent.getAttributeNode(childName[1:])) + else: + raise Exception, "Attribute cannot have a child node '%s'" % xpath + if onlyOne: + break + return returnNodes + + def removeNode (self, xpath): + matchingNodes = self.getNodes(xpath, False) + for matchingNode in matchingNodes: + if matchingNode.nodeType == Node.ATTRIBUTE_NODE: + ownerElement = matchingNode.ownerElement + ownerElement.removeAttribute(matchingNode.nodeName) + else: + parentNode = matchingNode.parentNode + parentNode.removeChild(matchingNode) + + # Print tabbed XML for this item, useful for debugging. + def printXml (self): + print ASCII_ENC (self.root.toprettyxml (), 'xmlcharrefreplace') [0] + + # return underlying minidom of XmlWrapper + def toXml (self): + return self.root.toxml () + + def nodeCount(self, xpath): + return len(self.getNodes(xpath)) + + def createNode (self, xpath, value): + newNodes = self._createNewNodes(xpath) + for matchingNode in newNodes: + if matchingNode.nodeType == Node.ATTRIBUTE_NODE: + matchingNode.nodeValue = value + else: + matchingNode.appendChild(self.document.createTextNode(value)) + + def setNode (self, xpath, value, createNew=False): + if createNew: + self.createNode(xpath, value) + else: + matchingNodes = self.getNodes(xpath, False) + for matchingNode in matchingNodes: + if matchingNode.nodeType == Node.ATTRIBUTE_NODE: + matchingNode.nodeValue = value + else: + # delete all text nodes + for child in matchingNode.childNodes: + if child.nodeType == Node.TEXT_NODE: + matchingNode.removeChild(child) + # add text node + matchingNode.appendChild(self.document.createTextNode(value)) + if len(matchingNodes) == 0: + self.createNode(xpath, value) + + def getSubtree(self, xpath): + return PropBagEx(self.getNodes(xpath, string=False) [0]) + + def newSubtree(self, xpath): + return PropBagEx(self._createNewNodes(xpath, onlyOne = True)[0]) + + # deprecated + def iterate(self, xpath): + return self.getSubtrees(xpath) + + def getSubtrees(self, xpath): + nodes = self.getNodes(xpath, string=False) + return [PropBagEx(x) for x in nodes] + + def nodeExists(self, xpath): + return self.nodeCount(xpath) > 0 + + def validateXpath(self, xpath): + return self.xpath.validateXpath(xpath) + +class XPath: + def __init__ (self): + self.debugLevel = 0 + self.testNode = parseString("").firstChild + + # selectNodes() returns list of strings or nodes for the given + # XPath relative to the given node + def selectNodes(self, xpath, curNode, asStrings = False): + self.validateXpath(xpath) + return self._selectNodes(xpath, asStrings, curNode, False, self.debugLevel) + + # validateXpath() validates the given XPath (lightweight) + def validateXpath(self, xpath): + self._selectNodes(xpath, False, self.testNode, True, self.debugLevel, self.debugLevel) + return True + + def _selectNodes(self, xpath, asStrings, curNode, validateOnly = False, dl = 0, sd = 0): + if dl > 1: + print " DEBUG", "".ljust(sd),"_selectNodes(<%s>, %s%s)" % (curNode.nodeName, xpath, ", validateOnly" if validateOnly else "") + + step, remainingXpath, delimiter = self.splitFirstOuter(xpath, ["/"], False, dl = dl, sd = sd + 1) + + stepNodename = step + stepPredicate = "" + stepIsAttribute = False + matchingNodes = [] + + i = step.find("[") + + if i != -1: + stepNodename = step[:i] + stepPredicate = step[i + 1:-1] + if stepPredicate == "": + raise Exception, "Empty predicate ('%s')" % step + + # XPath validation only + if validateOnly: + if stepNodename not in ["", ".", "..", "*", "text()", "node()"]: + if stepNodename.startswith("@"): + nodename = stepNodename[1:] + else: + nodename = stepNodename + + # validate nodename + if nodename not in ["*", "node()"]: + if not all(c in ascii_letters+'-_.0123456789' for c in nodename) or nodename[0] in '-_.0123456789': + raise Exception, "Invalid token in xpath ('%s')" % stepNodename + + # evaludate predicate to validate it + if stepPredicate != "": + self.evaluateCondition(stepPredicate, curNode, validateOnly, 0, 0, dl = dl, sd = sd + 1) + + # if more xpath left recurse + if remainingXpath != "": + self._selectNodes(remainingXpath, False, curNode, validateOnly, dl = dl, sd = sd + 1) + + if dl >= 1: + print " DEBUG", "".ljust(sd),"_selectNodes(<%s>, %s) -> %s" % (curNode.nodeName, xpath, "VALID") + return [] + + # XPath Processing + + # check if attribute + if stepNodename.startswith("@") or stepNodename == "node()": + if stepNodename.startswith("@"): + attrNodename = stepNodename[1:] + else: + attrNodename = stepNodename + + if curNode.nodeType == Node.ELEMENT_NODE: + if curNode.hasAttribute(attrNodename): + # match on attribute name + matchingNodes.append(curNode.getAttributeNode(attrNodename)) + + elif attrNodename == "*" or attrNodename == "node()": + # wildcard so iterate through attributes and do predicate test on each + predicateMatch = False + i = 0 + for key in curNode.attributes.keys(): + i += 1 + # check for predicate + if stepPredicate == "": + # no predicate + predicateMatch = True + else: + # get count of attributes (for last()) + childNodeCount = len(curNode.attributes) + + # evaluate attriute wildcard predicate + predicateMatch = self.evaluateCondition(stepPredicate, curNode.attributes[key], validateOnly, i, childNodeCount, dl = dl, sd = sd + 1) + + # append node to return if matching + if predicateMatch: + matchingNodes.append(curNode.attributes[key]) + + # check if named element + if stepNodename not in ["", ".", "..", "text()"] and not stepNodename.startswith("@"): + + # get count of child nodes (for last()) + childNodeCount = 0 + for childNode in curNode.childNodes: + if childNode.nodeType == Node.ELEMENT_NODE and (childNode.nodeName == stepNodename or stepNodename == "*"): + childNodeCount += 1 + + # iterate through elements + i = 0 + for childNode in curNode.childNodes: + if childNode.nodeType == Node.ELEMENT_NODE: + predicateMatch = False + + # only include child elements whose names match + if stepNodename in [childNode.nodeName, "*", "node()"]: + i += 1 + # check for predicate + if stepPredicate == "": + # no predicate + predicateMatch = True + else: + # evaluate predicate + predicateMatch = self.evaluateCondition(stepPredicate, childNode, validateOnly, i, childNodeCount, dl = dl, sd = sd + 1) + if predicateMatch: + if remainingXpath != "": + if remainingXpath.startswith("/"): + + # xpath was split on a double slash + remainingXpath = remainingXpath[1:] + matchingNodes += self.queryAllChildElements(remainingXpath, childNode, validateOnly) + + # more relative xpath remaining so recurse _selectNodes() and include results for returning + matchingNodes += self._selectNodes(remainingXpath, asStrings=False, curNode = childNode, validateOnly=validateOnly, dl = dl, sd = sd + 1) + + else: + # no xpath left so include child node for returning + matchingNodes.append(childNode) + + # check if empty string + elif stepNodename == "": + if remainingXpath != "": + # absolute path + curNode = curNode.ownerDocument + if remainingXpath.startswith("/"): + # xpath was split on a double slash + remainingXpath = remainingXpath[1:] + matchingNodes += self.queryAllChildElements(remainingXpath, curNode, validateOnly) + + matchingNodes += self._selectNodes(remainingXpath, asStrings=False, curNode = curNode, validateOnly=validateOnly, dl = dl, sd = sd + 1) + else: + # empty xpath + matchingNodes.append(curNode) + + # check if self or parent + elif stepNodename == ".": + matchingNodes.append(curNode) + elif stepNodename == "..": + if remainingXpath != "": + # more relative xpath remaining so recurse _selectNodes() + matchingNodes += self._selectNodes(remainingXpath, asStrings=False, curNode = curNode.parentNode, validateOnly=validateOnly, dl = dl, sd = sd + 1) + + else: + matchingNodes.append(curNode.parentNode) + + # check if text() node test + elif stepNodename == "text()": + for child in curNode.childNodes: + if child.nodeType == Node.TEXT_NODE: + matchingNodes.append(child) + + # return matches as nodes or strings + if not asStrings: + returnValues = matchingNodes + else: + # determine node values depending on elements or attributes + returnValues = [] + for node in matchingNodes: + if node.nodeType == Node.ATTRIBUTE_NODE or node.nodeType == Node.TEXT_NODE: + returnValues.append(node.nodeValue) + else: + # concatenate all the text nodes + value = "" + for child in node.childNodes: + if child.nodeType == Node.TEXT_NODE: + value += child.nodeValue + returnValues.append(value) + + if dl >= 1: + print " DEBUG", "".ljust(sd),"_selectNodes(<%s>, %s) -> %s" % (curNode.nodeName, xpath, returnValues) + return returnValues + + def queryAllChildElements(self, xpath, curNode, validateOnly, dl = 0, sd = 0): + result = [] + for childNode in curNode.childNodes: + # check if childNode is an element + if childNode.nodeType == Node.ELEMENT_NODE: + result += self._selectNodes(xpath, asStrings=False, curNode = childNode, validateOnly=validateOnly, dl = dl, sd = sd + 1) + + # recurse queryAllChildElements() + result += self.queryAllChildElements(xpath, childNode, dl, sd) + + return result + + def reverseString(self, string): + result = "" + for char in string: + result = char + result + return result + + def splitFirstOuter(self, string, delimiters, reverse=False, dl = 0, sd = 0): + lhs = "" + rhs = "" + foundDelimiter = "" + unclosedBrackets = 0 + unclosedParentheses = 0 + inDoubleQuote = False + inSingleQuote = False + if reverse: + string = self.reverseString(string) + for i, char in enumerate(string): + if char == "'" and not inDoubleQuote and string[i - 1] != "\\": + if inSingleQuote: + inSingleQuote = False + else: + inSingleQuote = True + if char == '"' and not inSingleQuote and string[i - 1] != "\\": + if inDoubleQuote: + inDoubleQuote = False + else: + inDoubleQuote = True + if not inSingleQuote and not inDoubleQuote: + if char == "[": + unclosedBrackets += 1 + if char == "]": + unclosedBrackets -= 1 + if char == "(": + unclosedParentheses += 1 + if char == ")": + unclosedParentheses -= 1 + + if unclosedBrackets == 0 and unclosedParentheses == 0: + for delimiter in delimiters: + if string[i:i + len(delimiter)].lower() == delimiter: + rhs = string[i + len(delimiter):] + foundDelimiter = delimiter + if foundDelimiter != "": + break + lhs += char + + if unclosedBrackets > 0: + raise Exception, "Missing ']' in '%s'" % string + if unclosedBrackets < 0: + raise Exception, "Extra ']' in '%s'" % string + if unclosedParentheses > 0: + raise Exception, "Missing ')' in '%s'" % string + if unclosedParentheses < 0: + raise Exception, "Extra ')' in '%s'" % string + if inDoubleQuote or inSingleQuote: + raise Exception, "Unterminated string '%s'" % string + + if reverse: + rhsTemp = self.reverseString(lhs) + lhs = self.reverseString(rhs) + rhs = rhsTemp + if dl >= 7: + if reverse: + string = self.reverseString(string) + print " DEBUG", "".ljust(sd),"splitFirstOuter('%s', %s) -> %s" % (string, delimiters, (lhs, rhs, foundDelimiter)) + return lhs.strip(), rhs.strip(), foundDelimiter.strip() + + def evaluateCondition(self, statement, curNode, validateOnly, nodePosition, sibCount, dl = 0, sd = 0): + if dl > 2: + print " DEBUG", "".ljust(sd),"evaluateCondition(%s)" % (statement) + lhs, rhs, operator = self.splitFirstOuter(statement, [" and ", " or "], False, dl, sd + 1) + + if lhs.strip()[0] == "(": + i = lhs.rfind(")") + if i == -1: + raise Exception, "Missing ')' " + lhs + # process statement in parantheses + lhsResult = self.evaluateCondition(lhs.strip()[1:i], curNode, validateOnly, nodePosition, sibCount, dl, sd + 1) + else: + # process statement + lhsResult, lhsResultType = self.evaluateComparison(lhs, curNode, validateOnly, nodePosition, sibCount, dl, sd + 1) + + # nodeset then return false if empty else true + if lhsResultType == "NODE_SET": + if len(lhsResult) > 0: + lhsResult = True + else: + lhsResult = False + + # int treat as node index + if lhsResultType == "NUMBER": + if lhsResult == nodePosition: + lhsResult = True + else: + lhsResult = False + + if rhs != "": + # recurse the righthand side + rhsResult = self.evaluateCondition(rhs, curNode, validateOnly, nodePosition, sibCount, dl = dl, sd = sd + 1) + if operator == "and": + result = lhsResult and rhsResult + else: + result = lhsResult or rhsResult + else: + result = lhsResult + if dl >= 2: + print " DEBUG", "".ljust(sd),"evaluateCondition(%s) -> %s" % (statement, result) + return result + + + def evaluateComparison(self, string, curNode, validateOnly, nodePosition, sibCount, dl = 0, sd = 0): + if dl > 3: + print " DEBUG", "".ljust(sd),"evaluateComparison(%s)" % (string) + lhs, rhs, operator = self.splitFirstOuter(string, ["=", "!=", "<", ">", "<=", ">="], False, dl = dl, sd = sd + 1) + lhsResult, lhsResultType = self.evaluateStatement(lhs, curNode, validateOnly, nodePosition, sibCount, dl, sd + 1) + + if rhs != "": + rhsResult, rhsResultType = self.evaluateStatement(rhs, curNode, validateOnly, nodePosition, sibCount, dl, sd + 1) + result = False, "BOOLEAN" + if lhsResultType == "NODE_SET" and rhsResultType == "NODE_SET": + for lhsValue in lhsResult: + for rhsValue in rhsResult: + result = self.compareValues(operator, lhsValue, "STRING", rhsValue, "STRING", dl, sd), "BOOLEAN" + if result[0]: + break + if result[0]: + break + elif lhsResultType == "NODE_SET": + for lhsValue in lhsResult: + result = self.compareValues(operator, lhsValue, "STRING", rhsResult, rhsResultType, dl, sd), "BOOLEAN" + if result[0]: + break + elif rhsResultType == "NODE_SET": + for rhsValue in rhsResult: + result = self.compareValues(operator, lhsResult, rhsResultType, rhsValue, "STRING", dl, sd), "BOOLEAN" + if result[0]: + break + else: + result = self.compareValues(operator, lhsResult, lhsResultType, rhsResult, rhsResultType, dl, sd), "BOOLEAN" + else: + result = lhsResult, lhsResultType + if dl >= 3: + print " DEBUG", "".ljust(sd),"evaluateComparison(%s) -> %s" % (string, result) + return result + + # function for evaluating against different operators + def compareValues(self, operator, lhsValue, lhsValueType, rhsValue, rhsValueType, dl = 0, sd = 0): + result = False + if rhsValueType == "NUMBER" and lhsValueType == "STRING": + try: + lhsValue = int(lhsValue) + except ValueError: + pass + elif rhsValueType == "STRING" and lhsValueType == "NUMBER": + try: + rhsValue = int(rhsValue) + except ValueError: + pass + if operator == "=" and lhsValue == rhsValue: + result = True + elif operator == "!=" and lhsValue != rhsValue: + result = True + elif operator == "<" and lhsValue < rhsValue: + result = True, "BOOLEAN" + elif operator == ">" and lhsValue > rhsValue: + result = True, "BOOLEAN" + elif operator == "<=" and lhsValue <= rhsValue: + result = True, "BOOLEAN" + elif operator == ">=" and lhsValue >= rhsValue: + result = True, "BOOLEAN" + if dl >= 4: + print " DEBUG", "".ljust(sd),"compareValues(%s %s %s) -> %s" % (lhsValue, operator, rhsValue, result) + return result + + def evaluateStatement(self, statement, curNode, validateOnly, nodePosition, sibCount, dl = 0, sd = 0): + if dl > 5: + print " DEBUG", "".ljust(sd),"evaluateStatement(%s)" % (statement) + lhs, rhs, operator = self.splitFirstOuter(statement, ["+", " - "], True, dl, sd + 1) + + # check for outer parentheses + if rhs.strip()[0] == "(": + i = rhs.rfind(")") + if i == -1: + raise Exception, "Missing ')' " + rhs + # process statement in parantheses + rhsResult, rhsResultType = self.evaluateStatement(rhs.strip()[1:i], curNode, validateOnly, nodePosition, sibCount, dl, sd + 1) + else: + rhsResult, rhsResultType = self.evaluateValue(rhs, curNode, validateOnly, nodePosition, sibCount, dl, sd + 1) + + if lhs != "": + # recurse the lefthand side + lhsResult, lhsResultType = self.evaluateStatement(lhs, curNode, validateOnly, nodePosition, sibCount, dl, sd + 1) + if operator == "+": + if lhsResultType == "NUMBER" and rhsResultType == "NUMBER": + result = lhsResult + rhsResult, "NUMBER" + else: + raise Exception, "Can only use (+) operator for numbers not %s and %s '%s'" % (lhsResultType, rhsResultType, statement) + else: + if lhsResultType == "NUMBER" and rhsResultType == "NUMBER": + result = lhsResult - rhsResult, "NUMBER" + else: + raise Exception, "Can only use (-) operator for numbers not %s and %s '%s'" % (lhsResultType, rhsResultType, statement) + else: + result = rhsResult, rhsResultType + if dl >= 5: + print " DEBUG", "".ljust(sd),"evaluateStatement(%s) -> %s" % (statement, result) + return result + + + def evaluateValue(self, value, curNode, validateOnly, nodePosition, sibCount, dl = 0, sd = 0): + if dl > 5: + print " DEBUG", "".ljust(sd),"evaluateValue(%s)" % (value) + + # integer + try: + integer = int(value) + evaluation = integer, "NUMBER" + except ValueError: + # string + if (value.strip()[0] == '"' and value.strip()[-1] == '"') or (value.strip()[0] == "'" and value.strip()[-1] == "'"): + evaluation = value.strip()[1:-1], "STRING" + # true() + elif value.strip().startswith("true("): + parameters = self.getFunctionParameters(value, [], curNode, validateOnly, nodePosition, sibCount, dl = dl, sd = sd + 1) + evaluation = True, "BOOLEAN" + # false() + elif value.strip().startswith("false("): + parameters = self.getFunctionParameters(value, [], curNode, validateOnly, nodePosition, sibCount, dl = dl, sd = sd + 1) + evaluation = False, "BOOLEAN" + # postition() + elif value.strip().startswith("position("): + parameters = self.getFunctionParameters(value, [], curNode, validateOnly, nodePosition, sibCount, dl = dl, sd = sd + 1) + evaluation = nodePosition, "NUMBER" + # last() + elif value.strip().startswith("last("): + parameters = self.getFunctionParameters(value, [], curNode, validateOnly, nodePosition, sibCount, dl = dl, sd = sd + 1) + evaluation = sibCount, "NUMBER" + # name() + elif value.strip().startswith("name("): + # find closing parenthesis + i = value.rfind(")") + if i == -1: + raise Exception, "Malformed function '%s'" % value + parameterString = value[value.find("(") + 1:i].strip() + if parameterString == "": + evaluation = curNode.nodeName, "STRING" + else: + nodes = self._selectNodes(parameterString, asStrings=False, curNode=curNode, validateOnly=validateOnly) + if len(nodes) > 0: + evaluation = nodes[0].nodeName, "STRING" + else: + evaluation = "", "STRING" + # string() + elif value.strip().startswith("string("): + parameters = self.getFunctionParameters(value, ["BOOLEAN|STRING|NUMBER|NODE_SET"], curNode, validateOnly, nodePosition, sibCount, dl = dl, sd = sd + 1) + evaluation = self.convertToString(parameters[0][0], parameters[0][1]), "STRING" + # upper-case() + elif value.strip().startswith("upper-case("): + parameters = self.getFunctionParameters(value, ["BOOLEAN|STRING|NUMBER|NODE_SET"], curNode, validateOnly, nodePosition, sibCount, dl = dl, sd = sd + 1) + string = self.convertToString(parameters[0][0], parameters[0][1]) + + evaluation = string.upper(), "STRING" + # lower-case() + elif value.strip().startswith("lower-case("): + parameters = self.getFunctionParameters(value, ["BOOLEAN|STRING|NUMBER|NODE_SET"], curNode, validateOnly, nodePosition, sibCount, dl = dl, sd = sd + 1) + string = self.convertToString(parameters[0][0], parameters[0][1]) + + evaluation = string.lower(), "STRING" + # substring() + elif value.strip().startswith("substring("): + parameters = self.getFunctionParameters(value, ["BOOLEAN|STRING|NUMBER|NODE_SET","NUMBER","NUMBER"], curNode, validateOnly, nodePosition, sibCount, minParams=2, dl = dl, sd = sd + 1) + string = self.convertToString(parameters[0][0], parameters[0][1]) + + # get substring depending on parameters supplied + if len(parameters) == 2: + evaluation = string[parameters[1][0] - 1:], "STRING" + elif len(parameters) == 3: + evaluation = string[parameters[1][0] - 1:parameters[1][0] - 1 + parameters[2][0]], "STRING" + # starts-with() + elif value.strip().startswith("starts-with("): + parameters = self.getFunctionParameters(value, ["BOOLEAN|STRING|NUMBER|NODE_SET","BOOLEAN|STRING|NUMBER|NODE_SET"], curNode, validateOnly, nodePosition, sibCount, minParams=2, dl = dl, sd = sd + 1) + string1 = self.convertToString(parameters[0][0], parameters[0][1]) + string2 = self.convertToString(parameters[1][0], parameters[1][1]) + if string2 != "": + evaluation = string1.startswith(string2), "BOOLEAN" + else: + evaluation = False, "BOOLEAN" + # ends-with() + elif value.strip().startswith("ends-with("): + parameters = self.getFunctionParameters(value, ["BOOLEAN|STRING|NUMBER|NODE_SET","BOOLEAN|STRING|NUMBER|NODE_SET"], curNode, validateOnly, nodePosition, sibCount, minParams=2, dl = dl, sd = sd + 1) + string1 = self.convertToString(parameters[0][0], parameters[0][1]) + string2 = self.convertToString(parameters[1][0], parameters[1][1]) + if string2 != "": + evaluation = string1.endswith(string2), "BOOLEAN" + else: + evaluation = False, "BOOLEAN" + # contains() + elif value.strip().startswith("contains("): + parameters = self.getFunctionParameters(value, ["BOOLEAN|STRING|NUMBER|NODE_SET","BOOLEAN|STRING|NUMBER|NODE_SET"], curNode, validateOnly, nodePosition, sibCount, minParams=2, dl = dl, sd = sd + 1) + string1 = self.convertToString(parameters[0][0], parameters[0][1]) + string2 = self.convertToString(parameters[1][0], parameters[1][1]) + if string2 != "": + evaluation = string2 in string1, "BOOLEAN" + else: + evaluation = False, "BOOLEAN" + # string-length() + elif value.strip().startswith("string-length("): + parameters = self.getFunctionParameters(value, ["BOOLEAN|STRING|NUMBER|NODE_SET"], curNode, validateOnly, nodePosition, sibCount, minParams=0, dl = dl, sd = sd + 1) + + if len(parameters) == 0: + if curNode.firstChild != None: + string = self.convertToString(curNode.firstChild.nodeValue, "STRING") + if string != None: + evaluation = len(string), "NUMBER" + else: + evaluation = 0, "NUMBER" + else: + evaluation = 0, "NUMBER" + else: + evaluation = len(self.convertToString(parameters[0][0], parameters[0][1])), "NUMBER" + # concat() + elif value.strip().startswith("concat("): + parameters = self.getFunctionParameters(value, ["BOOLEAN|STRING|NUMBER|NODE_SET","*"], curNode, validateOnly, nodePosition, sibCount, minParams=2, dl = dl, sd = sd + 1) + string = "" + for parameter in parameters: + string += self.convertToString(parameter[0], parameter[1]) + evaluation = string, "STRING" + # not() + elif value.strip().startswith("not("): + parameters = self.getFunctionParameters(value, ["BOOLEAN"], curNode, validateOnly, nodePosition, sibCount, dl = dl, sd = sd + 1) + evaluation = not parameters[0][0], "BOOLEAN" + else: + # + evaluation = self._selectNodes(value, asStrings=True, curNode=curNode, validateOnly=validateOnly), "NODE_SET" + if dl >= 5: + print " DEBUG", "".ljust(sd),"evaluateValue(%s) -> %s" % (value, evaluation) + return evaluation + + def convertToString(self, value, valueType): + if valueType == "STRING": + convertedValue = value + elif valueType == "NODE_SET": + convertedValue = value[0] if len(value) > 0 else "" + elif valueType == "NUMBER": + convertedValue = str(value) + elif valueType == "BOOLEAN": + convertedValue = "true" if value else "false" + else: + raise Exception, "Cannot convert type %s '%s'" %(valueType, value) + + return convertedValue + + def getFunctionParameters(self, string, paramTypes, curNode, validateOnly, nodePosition, sibCount, minParams=-1, dl = 0, sd = 0): + if dl > 6: + print " DEBUG", "".ljust(sd),"getFunctionParameters(%s, %s)" % (string, paramTypes) + + # find closing parenthesis + i = string.rfind(")") + if i == -1: + raise Exception, "Malformed function '%s'" % string + + # split out parameters + parameterString = string[string.find("(") + 1:i] + parameters = [] + while parameterString != "": + param, parameterString, operator = self.splitFirstOuter(parameterString, [","], False, dl = dl, sd = sd + 1) + parameters.append((param, "")) + + # if second paramTypes = "*" this indicates any number of the first allowed type above + # the minimum is allowed so modify paramTypes accordingly + if len(paramTypes) >= 2 and paramTypes[1] == "*": + allowedType = paramTypes[0] + paramTypes = [] + for j in range(len(parameters)): + paramTypes.append(allowedType) + + # check number of parameters + if minParams == -1: + minParams = len(paramTypes) + if len(parameters) < minParams or len(parameters) > len(paramTypes): + if len(paramTypes) == 0: + msg = "Expecting no parameters" + elif len(paramTypes) == 1: + msg = "Expecting only 1 parameter" + elif len(paramTypes) == minParams: + msg = "Expecting exactly %s parameters" % minParams + else: + msg = "Expecting between %s and %s parameters" % (minParams, len(paramTypes)) + raise Exception, "%s '%s'" % (msg, string) + + # evaluate parameters + for i, parameter in enumerate(parameters): + allowedTypes = paramTypes[i].split("|") + + if [pt for pt in allowedTypes if pt in ["NUMBER", "STRING", "NODE_SET"]]: + # process int, string or xpath parameter + parameters[i] = self.evaluateStatement(parameter[0], curNode, validateOnly, nodePosition, sibCount, dl = dl, sd = sd + 1) + + # verify parameter types match expected + if parameters[i][1] not in allowedTypes: + raise Exception, "Incorrect parameter type %s in '%s' expecting one of the following %s" % (parameters[i][1], string, allowedTypes) + + elif "BOOLEAN" in allowedTypes: + # process boolean parameter + parameters[i] = self.evaluateCondition(parameter[0], curNode, validateOnly, nodePosition, sibCount, dl = dl, sd = sd + 1), "BOOLEAN" + if dl >= 6: + print " DEBUG", "".ljust(sd),"getFunctionParameters(%s, %s) -> %s" % (string, paramTypes, parameters) + return parameters + + +if __name__ == "__main__": + import doctest + doctest.testmod() diff --git a/package-mac/source/ebi/fileopen.png b/package-mac/source/ebi/fileopen.png new file mode 100644 index 0000000..69dd8d4 Binary files /dev/null and b/package-mac/source/ebi/fileopen.png differ diff --git a/package-mac/source/ebi/filesave.png b/package-mac/source/ebi/filesave.png new file mode 100644 index 0000000..22ff495 Binary files /dev/null and b/package-mac/source/ebi/filesave.png differ diff --git a/package-mac/source/ebi/gtk-help.png b/package-mac/source/ebi/gtk-help.png new file mode 100644 index 0000000..f25fc3f Binary files /dev/null and b/package-mac/source/ebi/gtk-help.png differ diff --git a/package-mac/source/ebi/gtk-stop.png b/package-mac/source/ebi/gtk-stop.png new file mode 100644 index 0000000..ab6808f Binary files /dev/null and b/package-mac/source/ebi/gtk-stop.png differ diff --git a/package-mac/source/ebi/options.png b/package-mac/source/ebi/options.png new file mode 100644 index 0000000..fdbbb71 Binary files /dev/null and b/package-mac/source/ebi/options.png differ diff --git a/package-mac/source/ebi/pause.png b/package-mac/source/ebi/pause.png new file mode 100644 index 0000000..254d2dd Binary files /dev/null and b/package-mac/source/ebi/pause.png differ