diff --git a/.gitignore b/.gitignore index 1485220..89a753f 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,5 @@ harness/var/goleveldb_data/LOCK harness/var/goleveldb_data/LOG + +*.zip diff --git a/app/naprr/naprr.go b/app/naprr/naprr.go new file mode 100644 index 0000000..2133aa0 --- /dev/null +++ b/app/naprr/naprr.go @@ -0,0 +1,94 @@ +package main + +import ( + "flag" + "github.com/nats-io/nats-streaming-server/server" + "github.com/nsip/nias2/naprr" + "log" + "net" + "os" + // "os/exec" + // "os/signal" + // "runtime" + // "time" +) + +var rewrite = flag.Bool("rewrite", false, "rewrite regenerates all reports without re-loading data") + +func main() { + + flag.Parse() + + wd, _ := os.Getwd() + log.Println("working directory:", wd) + + if !*rewrite { + log.Println("removing old files...") + clearNSSWorkingDirectory() + } + + log.Println("Launching stream server...") + ss := launchNatsStreamingServer() + defer ss.Shutdown() + + if !*rewrite { + log.Println("Starting data ingest...") + + di := naprr.NewDataIngest() + di.Run() + + log.Println("Generating report data...") + rb := naprr.NewReportBuilder() + rb.Run() + } + + log.Println("Writing report files...") + rw := naprr.NewReportWriter() + rw.Run() + + log.Println("Done.") + + // runtime.Goexit() +} + +func clearNSSWorkingDirectory() { + + // remove existing logs and recreate the directory + err := os.RemoveAll("nss") + err = os.Mkdir("nss", os.ModePerm) + if err != nil { + log.Println("Error trying to remove nss working directory") + } +} + +func launchNatsStreamingServer() *server.StanServer { + + stanOpts := server.GetDefaultOptions() + + stanOpts.ID = "nap-rr" + stanOpts.MaxChannels = 30000 + stanOpts.MaxMsgs = 2000000 + stanOpts.MaxBytes = 0 //unlimited + stanOpts.MaxSubscriptions = 10000 + + stanOpts.StoreType = "FILE" + stanOpts.FilestoreDir = "nss" + // stanOpts.Debug = true + + ss := server.RunServerWithOpts(stanOpts, nil) + + return ss + +} + +// getAvailPort asks the OS for an unused port. +// There's a race here, where the port could be grabbed by someone else +// before the caller gets to Listen on it, but in practice such races +// are rare. Uses net.Listen("tcp", ":0") to determine a free port, then +// releases it back to the OS with Listener.Close(). +func getAvailPort() int { + l, _ := net.Listen("tcp", ":0") + r := l.Addr() + l.Close() + return r.(*net.TCPAddr).Port +} diff --git a/app/naprr/templates/codeframe_hdr.tmpl b/app/naprr/templates/codeframe_hdr.tmpl new file mode 100644 index 0000000..65a9b80 --- /dev/null +++ b/app/naprr/templates/codeframe_hdr.tmpl @@ -0,0 +1,20 @@ +{{- "Test Name"}}, +{{- "Test Type"}}, +{{- "Test Level"}}, +{{- "Domain"}}, +{{- "Item ID"}}, +{{- "Item Subdomain"}}, +{{- "Item Max. Score"}}, +{{- "Item Correct Ans."}}, +{{- "Item Descriptor"}}, +{{- "Item Examplar Link"}}, +{{- "Item Difficulty"}}, +{{- "Item Type"}}, +{{- "Item Weight"}}, +{{- "Item Marking Type"}}, +{{- "Item loc. in Testlet"}}, +{{- "Testlet Node"}}, +{{- "Testlet loc. in Test"}}, +{{- "Testlet Name"}}, +{{- "Testlet ID"}}, +{{- "Testlet Max. Score"}} diff --git a/app/naprr/templates/codeframe_row.tmpl b/app/naprr/templates/codeframe_row.tmpl new file mode 100644 index 0000000..76b1867 --- /dev/null +++ b/app/naprr/templates/codeframe_row.tmpl @@ -0,0 +1,20 @@ +{{ printf "\"%s\"" .Test.TestContent.TestName}}, +{{- .Test.TestContent.TestType}}, +{{- .Test.TestContent.TestLevel}}, +{{- .Test.TestContent.TestDomain}}, +{{- .Item.TestItemContent.NAPTestItemLocalId}}, +{{- .Item.TestItemContent.Subdomain}}, +{{- .Item.TestItemContent.MaximumScore}}, +{{- .Item.TestItemContent.CorrectAnswer}}, +{{- .Item.TestItemContent.ItemDescriptor}}, +{{- .Item.TestItemContent.ExemplarURL}}, +{{- .Item.TestItemContent.ItemDifficulty}}, +{{- .Item.TestItemContent.ItemType}}, +{{- "Item Weight"}}, +{{- .Item.TestItemContent.MarkingType}}, +{{- .GetItemLocationInTestlet .Item.ItemID}}, +{{- .Testlet.TestletContent.Node}}, +{{- .Testlet.TestletContent.LocationInStage}}, +{{- .Testlet.TestletContent.TestletName}}, +{{- .Testlet.TestletContent.LocalId}}, +{{- .Testlet.TestletContent.TestletMaximumScore}} diff --git a/app/naprr/templates/codeframe_writing_hdr.tmpl b/app/naprr/templates/codeframe_writing_hdr.tmpl new file mode 100644 index 0000000..3479763 --- /dev/null +++ b/app/naprr/templates/codeframe_writing_hdr.tmpl @@ -0,0 +1,45 @@ +{{- "Test Name"}}, +{{- "Test Type"}}, +{{- "Test Level"}}, +{{- "Domain"}}, +{{- "Item ID"}}, +{{- "Item Subdomain"}}, +{{- "Item Max. Score"}}, +{{- "Item Correct Ans."}}, +{{- "Item Descriptor"}}, +{{- "Item Examplar Link"}}, +{{- "Item Difficulty"}}, +{{- "Item Type"}}, +{{- "Item Weight"}}, +{{- "Item Marking Type"}}, +{{- "Item loc. in Testlet"}}, +{{- "Testlet Node"}}, +{{- "Testlet loc. in Test"}}, +{{- "Testlet Name"}}, +{{- "Testlet ID"}}, +{{- "Testlet Max. Score"}}, +{{- "Genre"}}, +{{- "Stimulus"}}, +{{- "Descriptor"}}, +{{- "Audience Descriptor"}}, +{{- "Audience Max"}}, +{{- "Ch./Sett. Descriptor"}}, +{{- "Ch./Sett. Max"}}, +{{- "Cohesion Descriptor"}}, +{{- "Cohesion Max"}}, +{{- "Ideas Descriptor"}}, +{{- "Ideas Max"}}, +{{- "Para. Descriptor"}}, +{{- "Para. Max"}}, +{{- "Persuasive Devices Descriptor"}}, +{{- "Persuasive Devices Max"}}, +{{- "Punctuation Descriptor"}}, +{{- "Punctuation Max"}}, +{{- "Sentence Structure Descriptor"}}, +{{- "Sentence Structure Max"}}, +{{- "Spelling Descriptor"}}, +{{- "Spelling Max"}}, +{{- "Text Structure Descriptor"}}, +{{- "Text Structure Max"}}, +{{- "Vocabulary Descriptor"}}, +{{- "Vocabulary Max"}} diff --git a/app/naprr/templates/codeframe_writing_row.tmpl b/app/naprr/templates/codeframe_writing_row.tmpl new file mode 100644 index 0000000..eb05f6a --- /dev/null +++ b/app/naprr/templates/codeframe_writing_row.tmpl @@ -0,0 +1,45 @@ +{{ printf "\"%s\"" .Test.TestContent.TestName}}, +{{- .Test.TestContent.TestType}}, +{{- .Test.TestContent.TestLevel}}, +{{- .Test.TestContent.TestDomain}}, +{{- .Item.TestItemContent.NAPTestItemLocalId}}, +{{- .Item.TestItemContent.Subdomain}}, +{{- .Item.TestItemContent.MaximumScore}}, +{{- .Item.TestItemContent.CorrectAnswer}}, +{{- .Item.TestItemContent.ItemDescriptor}}, +{{- .Item.TestItemContent.ExemplarURL}}, +{{- .Item.TestItemContent.ItemDifficulty}}, +{{- .Item.TestItemContent.ItemType}}, +{{- "Item Weight"}}, +{{- .Item.TestItemContent.MarkingType}}, +{{- .GetItemLocationInTestlet .Item.ItemID}}, +{{- .Testlet.TestletContent.Node}}, +{{- .Testlet.TestletContent.LocationInStage}}, +{{- .Testlet.TestletContent.TestletName}}, +{{- .Testlet.TestletContent.LocalId}}, +{{- .Testlet.TestletContent.TestletMaximumScore}}, +{{- .Item.TestItemContent.WritingGenre}}, +{{- "Writing Stimulus"}}, +{{- "Text Descriptor"}}, +{{- .GetWritingRubricDescriptor "Audience"}}, +{{- .GetWritingRubricMax "Audience"}}, +{{- .GetWritingRubricDescriptor "Character and Setting"}}, +{{- .GetWritingRubricMax "Character and Setting"}}, +{{- .GetWritingRubricDescriptor "Cohesion"}}, +{{- .GetWritingRubricMax "Cohesion"}}, +{{- .GetWritingRubricDescriptor "Ideas"}}, +{{- .GetWritingRubricMax "Ideas"}}, +{{- .GetWritingRubricDescriptor "Paragraphing"}}, +{{- .GetWritingRubricMax "Paragraphing"}}, +{{- .GetWritingRubricDescriptor "Persuasive Devices"}}, +{{- .GetWritingRubricMax "Persuasive Devices"}}, +{{- .GetWritingRubricDescriptor "Punctuation"}}, +{{- .GetWritingRubricMax "Punctuation"}}, +{{- .GetWritingRubricDescriptor "Sentence Structure"}}, +{{- .GetWritingRubricMax "Sentence Structure"}}, +{{- .GetWritingRubricDescriptor "Spelling"}}, +{{- .GetWritingRubricMax "Spelling"}}, +{{- .GetWritingRubricDescriptor "Text Structure"}}, +{{- .GetWritingRubricMax "Text Structure"}}, +{{- .GetWritingRubricDescriptor "Vocabulary"}}, +{{- .GetWritingRubricMax "Vocabulary"}} diff --git a/app/naprr/templates/domainscore_hdr.tmpl b/app/naprr/templates/domainscore_hdr.tmpl new file mode 100644 index 0000000..c398d5f --- /dev/null +++ b/app/naprr/templates/domainscore_hdr.tmpl @@ -0,0 +1,7 @@ +{{- "Test Name"}}, +{{- "PSI"}}, +{{- "Scaled Score Value"}}, +{{- "Raw Score"}}, +{{- "Scaled Score Stnadard Error"}}, +{{- "Student Domain Band"}}, +{{- "Student Proficiency"}} diff --git a/app/naprr/templates/domainscore_row.tmpl b/app/naprr/templates/domainscore_row.tmpl new file mode 100644 index 0000000..2ce8f90 --- /dev/null +++ b/app/naprr/templates/domainscore_row.tmpl @@ -0,0 +1,7 @@ +{{ printf "\"%s\"" .Test.TestContent.TestName}}, +{{- .Response.PSI}}, +{{- .Response.DomainScore.ScaledScoreValue}}, +{{- .Response.DomainScore.RawScore}}, +{{- .Response.DomainScore.ScaledScoreStandardError}}, +{{- .Response.DomainScore.StudentDomainBand}}, +{{- .Response.DomainScore.StudentProficiency}} diff --git a/app/naprr/templates/participation_hdr.tmpl b/app/naprr/templates/participation_hdr.tmpl new file mode 100644 index 0000000..3f7caf4 --- /dev/null +++ b/app/naprr/templates/participation_hdr.tmpl @@ -0,0 +1,25 @@ +{{- "DOB"}}, +{{- "Sex"}}, +{{- "Ind. Status"}}, +{{- "LBOTE"}}, +{{- "YearLevel"}}, +{{- "ASL School ID"}}, +{{- "HomeSchooled"}}, +{{- "Parent1 School Education"}}, +{{- "Parent1 NonSchool Education"}}, +{{- "Parent 1 Occupation"}}, +{{- "Parent2 School Education"}}, +{{- "Parent2 NonSchool Education"}}, +{{- "Parent2Occupation"}}, +{{- "PlatformID"}}, +{{- "School Name"}}, +{{- "School Postcode"}}, +{{- "School State"}}, +{{- "School Suburb"}}, +{{- "School Geolocation"}}, +{{- "SchoolSector"}}, +{{- "Writing"}}, +{{- "Reading"}}, +{{- "Spelling"}}, +{{- "Grammar and Punctuation"}}, +{{- "Numeracy"}} diff --git a/app/naprr/templates/participation_row.tmpl b/app/naprr/templates/participation_row.tmpl new file mode 100644 index 0000000..4375f8b --- /dev/null +++ b/app/naprr/templates/participation_row.tmpl @@ -0,0 +1,27 @@ +{{ .Student.BirthDate}}, +{{- .Student.Sex}}, +{{- .Student.IndigenousStatus}}, +{{- .Student.LBOTE}}, +{{- .Student.YearLevel}}, +{{- .Student.ASLSchoolId}}, +{{- .Student.HomeSchooledStudent}}, +{{- .Student.Parent1SchoolEducation}}, +{{- .Student.Parent1NonSchoolEducation}}, +{{- .Student.Parent1Occupation}}, +{{- .Student.Parent2SchoolEducation}}, +{{- .Student.Parent2NonSchoolEducation}}, +{{- .Student.Parent2Occupation}}, +{{- .Student.GetOtherId "NAPPlatformStudentId"}}, +{{- printf "\"%s\"" .School.SchoolName}}, +{{- with $sci := index .School.SchoolContactList.SchoolContact 0 }} +{{- $sci.ContactInfo.Address.PostalCode}}, +{{- $sci.ContactInfo.Address.StateProvince}}, +{{- $sci.ContactInfo.Address.City}}, +{{- end}} +{{- .School.SchoolGeographicLocation}}, +{{- .School.SchoolSector}}, +{{- index .Summary "Writing"}}, +{{- index .Summary "Reading"}}, +{{- index .Summary "Spelling"}}, +{{- index .Summary "Grammar and Punctuation"}}, +{{- index .Summary "Numeracy"}} diff --git a/app/naprr/templates/score_summary_hdr.tmpl b/app/naprr/templates/score_summary_hdr.tmpl new file mode 100644 index 0000000..249f995 --- /dev/null +++ b/app/naprr/templates/score_summary_hdr.tmpl @@ -0,0 +1,7 @@ +{{- "Test Name"}}, +{{- "ASL School ID"}}, +{{- "Domain National Average"}}, +{{- "Domain School Average"}}, +{{- "Domain Jurisdiction Average"}}, +{{- "Domain Top National 60%"}}, +{{- "Domain Bottom National 60%"}} diff --git a/app/naprr/templates/score_summary_row.tmpl b/app/naprr/templates/score_summary_row.tmpl new file mode 100644 index 0000000..e7b168f --- /dev/null +++ b/app/naprr/templates/score_summary_row.tmpl @@ -0,0 +1,7 @@ +{{ printf "\"%s\"" .Test.TestContent.TestName}}, +{{- .Summ.SchoolACARAId}}, +{{- .Summ.DomainNationalAverage}}, +{{- .Summ.DomainSchoolAverage}}, +{{- .Summ.DomainJurisdictionAverage}}, +{{- .Summ.DomainTopNational60Percent}}, +{{- .Summ.DomainBottomNational60Percent}} diff --git a/app/napval/napval.go b/app/napval/napval.go index 190f9f2..f20eaec 100644 --- a/app/napval/napval.go +++ b/app/napval/napval.go @@ -2,7 +2,6 @@ package main import ( - "github.com/nsip/nias2/lib" "github.com/nsip/nias2/napval" "log" "runtime" diff --git a/build_naprr.sh b/build_naprr.sh new file mode 100755 index 0000000..2942b13 --- /dev/null +++ b/build_naprr.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +set -e + +CWD=`pwd` + +# code is required for build, but no longer built as separate server +# is launched from within the naprr application +# echo "Downloading nats-streaming-server" +# go get github.com/nats-io/nats-streaming-server + +do_build() { + mkdir -p $OUTPUT + cd $CWD + cd ./app/naprr + go get + GOOS="$GOOS" GOARCH="$GOARCH" go build -ldflags="$LDFLAGS" -o $OUTPUT/$HARNESS + cd .. + rsync -a naprr/in naprr/templates $OUTPUT/ +} + +# do_shells() { +# cd $CWD +# cp bin/gonias.sh $OUTPUT/ +# cp bin/stopnias.sh $OUTPUT/ +# } + +# do_bats() { +# cd $CWD +# cp bin/gonias.bat $OUTPUT/ +# cp bin/stopnias.bat $OUTPUT/ +# } + +# do_upx() { +# upx $OUTPUT/$GNATS +# upx $OUTPUT/$HARNESS +# } + +# do_goupx() { +# goupx $OUTPUT/$GNATS +# goupx $OUTPUT/$HARNESS +# } + +do_zip() { + cd $OUTPUT + cd .. + zip -qr ../$ZIP naprr + cd $CWD +} + +build_mac64() { + # MAC OS X (64 only) + echo "Building Mac binaries..." + GOOS=darwin + GOARCH=amd64 + LDFLAGS="-s -w" + OUTPUT=$CWD/build/Mac/naprr + # GNATS=nats-streaming-server + HARNESS=naprr + ZIP=nias-naprr-Mac.zip + do_build + #do_upx + # do_shells + do_zip + echo "...all Mac binaries built..." +} + + +build_windows64() { + # WINDOWS 64 + echo "Building Windows64 binaries..." + GOOS=windows + GOARCH=amd64 + LDFLAGS="-s -w" + OUTPUT=$CWD/build/Win64/naprr + # GNATS=nats-streaming-server.exe + HARNESS=naprr.exe + ZIP=nias-naprr-Win64.zip + do_build + #do_upx + # do_bats + do_zip + echo "...all Windows64 binaries built..." +} + +build_windows32() { + # WINDOWS 32 + echo "Building Windows32 binaries..." + GOOS=windows + GOARCH=386 + LDFLAGS="-s -w" + OUTPUT=$CWD/build/Win32/naprr + # GNATS=nats-streaming-server.exe + HARNESS=naprr.exe + ZIP=nias-naprr-Win32.zip + do_build + #do_upx + # do_bats + do_zip + echo "...all Windows32 binaries built..." +} + +build_linux64() { + # LINUX 64 + echo "Building Linux64 binaries..." + GOOS=linux + GOARCH=amd64 + LDFLAGS="-s -w" + OUTPUT=$CWD/build/Linux64/naprr + # GNATS=nats-streaming-server + HARNESS=naprr + ZIP=nias-naprr-Linux64.zip + do_build + #do_goupx + # do_shells + do_zip + echo "...all Linux64 binaries built..." +} + +build_linux32() { + # LINUX 32 + echo "Building Linux32 binaries..." + GOOS=linux + GOARCH=386 + LDFLAGS="-s -w" + OUTPUT=$CWD/build/Linux32/naprr + # GNATS=nats-streaming-server + HARNESS=naprr + ZIP=nias-naprr-Linux32.zip + do_build + #do_goupx + # do_shells + do_zip + echo "...all Linux32 binaries built..." +} + +# TODO ARM +# GOOS=linux GOARCH=arm GOARM=7 go build -o $CWD/build/LinuxArm7/go-nias/aggregator + +build_mac64 +build_windows64 +build_windows32 +build_linux64 +build_linux32 + diff --git a/lib/transactiontracker.go b/lib/transactiontracker.go index c9db1ee..78777e6 100644 --- a/lib/transactiontracker.go +++ b/lib/transactiontracker.go @@ -33,13 +33,16 @@ var sizeMutex = &sync.Mutex{} // TransactionTracker simple strucuture to capture details of // transactions in system: tx size and tx progress type TransactionTracker struct { - C *nats.EncodedConn + C *nats.EncodedConn + ReportInterval int } // create a TransactionTracker -func NewTransactionTracker(TxReportInterval int, cfg NATSConfig) *TransactionTracker { - report_interval = TxReportInterval - return &TransactionTracker{C: CreateNATSConnection(cfg)} +// progress interval is how many messages between status updates +// eg. report status every 500 messages +func NewTransactionTracker(report_interval int, cfg NATSConfig) *TransactionTracker { + + return &TransactionTracker{C: CreateNATSConnection(cfg), ReportInterval: report_interval} } @@ -94,7 +97,7 @@ func (tt *TransactionTracker) GetStatusReport(txID string) (significantChange bo Size: size} // if progress < size but mod interval - if (progress % report_interval) == 0 { + if (progress % tt.ReportInterval) == 0 { sigChange = true } diff --git a/naprr/data_ingest.go b/naprr/data_ingest.go new file mode 100644 index 0000000..08d1a7c --- /dev/null +++ b/naprr/data_ingest.go @@ -0,0 +1,312 @@ +package naprr + +import ( + goxml "encoding/xml" + "github.com/nats-io/go-nats-streaming" + "github.com/nsip/nias2/lib" + "github.com/nsip/nias2/xml" + "log" + "path/filepath" + "sync" +) + +type DataIngest struct { + sc stan.Conn + ge GobEncoder + sr *StreamReader +} + +func NewDataIngest() *DataIngest { + di := DataIngest{sc: CreateSTANConnection(), ge: GobEncoder{}, sr: NewStreamReader()} + return &di +} + +func (di *DataIngest) Run() { + + xmlFiles := parseXMLFileDirectory() + + var wg sync.WaitGroup + + for _, xmlFile := range xmlFiles { + wg.Add(1) + go di.ingestResultsFile(xmlFile, &wg) + } + + wg.Wait() + + di.finaliseTransactions() + + di.sc.Close() + + log.Println("All data files read, ingest complete.") +} + +func parseXMLFileDirectory() []string { + + files, _ := filepath.Glob("./in/*.zip") + + if len(files) == 0 { + log.Fatalln("No results data zip files found in input folder.") + } + + return files + +} + +func (di *DataIngest) ingestResultsFile(resultsFilePath string, wg *sync.WaitGroup) { + + // create a connection to the streaming server + log.Println("Connecting to STAN server...") + + // map to hold student-school links temporarily + // so student responses can be assigned to correct schools + ss_link := make(map[string]string) + + // simple list of schools + // schools := make([]SchoolDetails, 0) + + // open the data file for streaming read + log.Printf("Opening results data file %s...", resultsFilePath) + xmlFile, err := OpenResultsFile(resultsFilePath) + if err != nil { + log.Fatalln("unable to open results data file: ", err) + } + + log.Println("Reading data file...") + + decoder := goxml.NewDecoder(xmlFile) + totalTests := 0 + totalTestlets := 0 + totalTestItems := 0 + totalTestScoreSummarys := 0 + totalEvents := 0 + totalResponses := 0 + totalCodeFrames := 0 + totalSchools := 0 + totalStudents := 0 + var inElement string + for { + // Read tokens from the XML document in a stream. + t, _ := decoder.Token() + if t == nil { + break + } + // Inspect the type of the token just read. + switch se := t.(type) { + case goxml.StartElement: + // If we just read a StartElement token + inElement = se.Name.Local + // ...handle by type + switch inElement { + case "NAPTest": + var t xml.NAPTest + decoder.DecodeElement(&t, &se) + gt, err := di.ge.Encode(t) + if err != nil { + log.Println("Unable to gob-encode nap test: ", err) + } + di.sc.Publish("meta", gt) + totalTests++ + + case "NAPTestlet": + var tl xml.NAPTestlet + decoder.DecodeElement(&tl, &se) + gtl, err := di.ge.Encode(tl) + if err != nil { + log.Println("Unable to gob-encode nap testlet: ", err) + } + di.sc.Publish("meta", gtl) + totalTestlets++ + + case "NAPTestItem": + var ti xml.NAPTestItem + decoder.DecodeElement(&ti, &se) + gti, err := di.ge.Encode(ti) + if err != nil { + log.Println("Unable to gob-encode nap test item: ", err) + } + di.sc.Publish("meta", gti) + totalTestItems++ + + case "NAPTestScoreSummary": + var tss xml.NAPTestScoreSummary + decoder.DecodeElement(&tss, &se) + gtss, err := di.ge.Encode(tss) + if err != nil { + log.Println("Unable to gob-encode nap test-score-summary: ", err) + } + di.sc.Publish(tss.SchoolACARAId, gtss) + totalTestScoreSummarys++ + + case "NAPEventStudentLink": + var e xml.NAPEvent + decoder.DecodeElement(&e, &se) + ge, err := di.ge.Encode(e) + if err != nil { + log.Println("Unable to gob-encode nap event link: ", err) + } + di.sc.Publish(e.SchoolID, ge) + totalEvents++ + + case "NAPStudentResponseSet": + var r xml.NAPResponseSet + decoder.DecodeElement(&r, &se) + gr, err := di.ge.Encode(r) + if err != nil { + log.Println("Unable to gob-encode student response set: ", err) + } + di.sc.Publish("responses", gr) + totalResponses++ + + case "NAPCodeFrame": + var cf xml.NAPCodeFrame + decoder.DecodeElement(&cf, &se) + gcf, err := di.ge.Encode(cf) + if err != nil { + log.Println("Unable to gob-encode nap codeframe: ", err) + } + di.sc.Publish("meta", gcf) + totalCodeFrames++ + + case "SchoolInfo": + var si xml.SchoolInfo + decoder.DecodeElement(&si, &se) + gsi, err := di.ge.Encode(si) + if err != nil { + log.Println("Unable to gob-encode schoolinfo: ", err) + } + di.sc.Publish(si.ACARAId, gsi) + // store school in local list + sd := SchoolDetails{ACARAId: si.ACARAId, SchoolName: si.SchoolName} + gsd, err := di.ge.Encode(sd) + if err != nil { + log.Println("Unable to gob-encode school-details: ", err) + } + di.sc.Publish("schools", gsd) + + totalSchools++ + + case "StudentPersonal": + var sp xml.RegistrationRecord + decoder.DecodeElement(&sp, &se) + gsp, err := di.ge.Encode(sp) + if err != nil { + log.Println("Unable to gob-encode studentpersonal: ", err) + } + // store linkage locally + ss_link[sp.RefId] = sp.ASLSchoolId + di.sc.Publish(sp.ASLSchoolId, gsp) + totalStudents++ + + } + default: + } + } + + log.Println("Data file read complete...") + log.Printf("Total tests: %d \n", totalTests) + log.Printf("Total codeframes: %d \n", totalCodeFrames) + log.Printf("Total testlets: %d \n", totalTestlets) + log.Printf("Total test items: %d \n", totalTestItems) + log.Printf("Total test score summaries: %d \n", totalTestScoreSummarys) + log.Printf("Total events: %d \n", totalEvents) + log.Printf("Total responses: %d \n", totalResponses) + log.Printf("Total schools: %d \n", totalSchools) + log.Printf("Total students: %d \n", totalStudents) + + log.Println("Finalising test metadata & responses...") + + // post end of stream message to responses queue + eot := lib.TxStatusUpdate{TxComplete: true} + geot, err := di.ge.Encode(eot) + if err != nil { + log.Println("Unable to gob-encode tx complete message: ", err) + } + di.sc.Publish("responses", geot) + di.sc.Publish("meta", geot) + + di.assignResponsesToSchools(ss_link) + + log.Println("response assignment complete") + + log.Printf("ingestion complete for %s", resultsFilePath) + + wg.Done() + +} + +func (di *DataIngest) assignResponsesToSchools(ss_link map[string]string) { + + log.Println("Assigning responses to schools...") + + // signal channel to notify asynch stan stream read is complete + txComplete := make(chan bool) + + // main message handling callback for the stan stream + mcb := func(m *stan.Msg) { + + // as we don't know message type ([]byte slice on wire) decode as interface + // then assert type dynamically + var m_if interface{} + err := di.ge.Decode(m.Data, &m_if) + if err != nil { + log.Println("message decoding error: ", err) + txComplete <- true + } + + switch t := m_if.(type) { + case xml.NAPResponseSet: + rs := m_if.(xml.NAPResponseSet) + studentId := rs.StudentID + schoolId := ss_link[studentId] + di.sc.Publish(schoolId, m.Data) + case lib.TxStatusUpdate: + txComplete <- true + default: + _ = t + log.Printf("unknown message type in response assign handler: %v", m_if) + } + // log.Printf("message decoded from stan is:\n\n %+v\n\n", msg) + + } + + sub, err := di.sc.Subscribe("responses", mcb, stan.DeliverAllAvailable()) + defer sub.Unsubscribe() + if err != nil { + log.Println("stan subsciption error - response assignment: ", err) + } + + <-txComplete + + log.Println("All reponses assigned to schools.") + +} + +func (di *DataIngest) finaliseTransactions() { + + log.Println("Finalising data read transactions...") + + // end of tx marker message + eot := lib.TxStatusUpdate{TxComplete: true} + geot, err := di.ge.Encode(eot) + if err != nil { + log.Println("Unable to gob-encode tx complete message: ", err) + } + + // finalise the known list of schools + di.sc.Publish("schools", geot) + + // then use the list to finalise each school data stream + schools := di.sr.GetSchoolDetails() + + for _, subslice := range schools { + for _, school := range subslice { + log.Println("finalising ", school) + di.sc.Publish(school.ACARAId, geot) + + } + } + + log.Println("All transactions finalised.") + +} diff --git a/naprr/naprr_encoding.go b/naprr/naprr_encoding.go new file mode 100644 index 0000000..83b1948 --- /dev/null +++ b/naprr/naprr_encoding.go @@ -0,0 +1,48 @@ +// encoding.go + +package naprr + +import ( + "bytes" + "encoding/gob" + "github.com/nsip/nias2/xml" +) + +// ensure all data types are registered with the encoder +func init() { + gob.Register(xml.NAPEvent{}) + gob.Register(xml.NAPCodeFrame{}) + gob.Register(xml.NAPResponseSet{}) + gob.Register(xml.NAPTest{}) + gob.Register(xml.NAPTestItem{}) + gob.Register(xml.NAPTestlet{}) + gob.Register(xml.NAPTestScoreSummary{}) + gob.Register(xml.RegistrationRecord{}) + gob.Register(xml.SchoolInfo{}) +} + +// GobEncoder is a Go specific GOB Encoder implementation for EncodedConn. +// This encoder will use the builtin encoding/gob to Marshal +// and Unmarshal most types, including structs. +type GobEncoder struct { + // Empty +} + +// Encode +// note: encoding the pointer is deliberate as forces +// encoded types to be interface{} +func (ge *GobEncoder) Encode(v interface{}) ([]byte, error) { + b := new(bytes.Buffer) + enc := gob.NewEncoder(b) + if err := enc.Encode(&v); err != nil { + return nil, err + } + return b.Bytes(), nil +} + +// Decode +func (ge *GobEncoder) Decode(data []byte, vPtr interface{}) (err error) { + dec := gob.NewDecoder(bytes.NewBuffer(data)) + err = dec.Decode(vPtr) + return +} diff --git a/naprr/naprr_naplandata.go b/naprr/naprr_naplandata.go new file mode 100644 index 0000000..8c234ee --- /dev/null +++ b/naprr/naprr_naplandata.go @@ -0,0 +1,26 @@ +// naprr_naplandata.go +package naprr + +import ( + "github.com/nsip/nias2/xml" + // "log" +) + +// structs to hold naplan meta-data that is the same for all schools + +type NAPLANData struct { + Tests map[string]xml.NAPTest + Testlets map[string]xml.NAPTestlet + Items map[string]xml.NAPTestItem + Codeframes map[string]xml.NAPCodeFrame +} + +func NewNAPLANData() *NAPLANData { + nd := NAPLANData{ + Tests: make(map[string]xml.NAPTest), + Testlets: make(map[string]xml.NAPTestlet), + Items: make(map[string]xml.NAPTestItem), + Codeframes: make(map[string]xml.NAPCodeFrame), + } + return &nd +} diff --git a/naprr/naprr_schooldata.go b/naprr/naprr_schooldata.go new file mode 100644 index 0000000..1a12bb9 --- /dev/null +++ b/naprr/naprr_schooldata.go @@ -0,0 +1,42 @@ +// schooldata.go +package naprr + +import ( + "github.com/nsip/nias2/xml" + "log" +) + +// brings together all school-specific data for analysis and reports +// + +type SchoolData struct { + SchoolInfos map[string]xml.SchoolInfo + Events map[string]xml.NAPEvent + ScoreSummaries map[string]xml.NAPTestScoreSummary + Responses map[string]xml.NAPResponseSet + Students map[string]xml.RegistrationRecord + ACARAId string +} + +func NewSchoolData(acaraid string) *SchoolData { + sd := SchoolData{ + SchoolInfos: make(map[string]xml.SchoolInfo), + Events: make(map[string]xml.NAPEvent), + ScoreSummaries: make(map[string]xml.NAPTestScoreSummary), + Responses: make(map[string]xml.NAPResponseSet), + Students: make(map[string]xml.RegistrationRecord), + ACARAId: acaraid, + } + return &sd +} + +func (sd *SchoolData) PrintSummary() { + log.Printf("\nSchool: %s", sd.ACARAId) + log.Printf("students: %d", len(sd.Students)) + log.Printf("events: %d", len(sd.Events)) + log.Printf("schoolinfos: %d", len(sd.SchoolInfos)) + log.Printf("school summaries: %d", len(sd.ScoreSummaries)) + log.Printf("responses: %d", len(sd.Responses)) + log.Printf("\n\n") + +} diff --git a/naprr/naprr_utils.go b/naprr/naprr_utils.go new file mode 100644 index 0000000..45b777c --- /dev/null +++ b/naprr/naprr_utils.go @@ -0,0 +1,81 @@ +package naprr + +import ( + "archive/zip" + "github.com/nats-io/go-nats-streaming" + "github.com/nats-io/nuid" + "io" + "log" + "os" +) + +// connection to the Streaming NATS server +func CreateSTANConnection() stan.Conn { + + clusterID := "nap-rr" + clientID := nuid.Next() + + sc, err := stan.Connect(clusterID, clientID) + if err != nil { + log.Fatalln("Cannot connect to STAN server: ", err) + } + + return sc + +} + +func isZipFile(fname string) bool { + + xmlZipFile, err := zip.OpenReader(fname) + if err != nil { + return false + } + defer xmlZipFile.Close() + + return true + +} + +func openDataFileZip(fname string) (io.Reader, error) { + + xmlZipFile, err := zip.OpenReader(fname) + if err != nil { + log.Println("Unable to open zip file: ", err) + return nil, err + } + // assume only one file in the archive + xmlFile, err := xmlZipFile.File[0].Open() + if err != nil { + return xmlFile, err + } + + return xmlFile, nil + +} + +func openDataFile(fname string) (io.Reader, error) { + + xmlFile, err := os.Open(fname) + if err != nil { + return xmlFile, err + } + return xmlFile, nil + +} + +func OpenResultsFile(fname string) (io.Reader, error) { + var xmlFile io.Reader + var ferr error + + if isZipFile(fname) { + xmlFile, ferr = openDataFileZip(fname) + } else { + xmlFile, ferr = openDataFile(fname) + } + if ferr != nil { + return xmlFile, ferr + } + + return xmlFile, ferr + +} diff --git a/naprr/naprr_xmlhelpers.go b/naprr/naprr_xmlhelpers.go new file mode 100644 index 0000000..e7f2620 --- /dev/null +++ b/naprr/naprr_xmlhelpers.go @@ -0,0 +1,158 @@ +package naprr + +import ( + "encoding/gob" + "github.com/nsip/nias2/xml" + "strings" +) + +func init() { + gob.Register(ParticipationDataSet{}) + gob.Register(EventInfo{}) + gob.Register(SchoolDetails{}) + gob.Register(ScoreSummaryDataSet{}) + gob.Register(ResponseDataSet{}) + gob.Register(CodeFrameDataSet{}) +} + +// convenience types for aggregating response information sets +// used in reporting and support types for sorting results. +// + +// aggregating type used for reporting domain scores +type ResponseDataSet struct { + Test xml.NAPTest + Response xml.NAPResponseSet +} + +// struct for sorting support +type ResponseComparator []ResponseDataSet + +// sort interface implementation for responsedatasets +func (resps ResponseComparator) Len() int { + return len(resps) +} + +// sort interface implementation for responsedatasets +func (resps ResponseComparator) Swap(i, j int) { + resps[i], resps[j] = resps[j], resps[i] +} + +// sort interface implementation for responsedatasets +func (resps ResponseComparator) Less(i, j int) bool { + return resps[i].Test.TestContent.TestName < resps[j].Test.TestContent.TestName +} + +// aggregating type used for producing school summary reports +type ScoreSummaryDataSet struct { + Test xml.NAPTest + Summ xml.NAPTestScoreSummary +} + +// struct for sorting support +type ScoreSummaryComparator []ScoreSummaryDataSet + +// sort interface implementation for summarydatasets +func (summs ScoreSummaryComparator) Len() int { + return len(summs) +} + +// sort interface implementation for summarydatasets +func (summs ScoreSummaryComparator) Swap(i, j int) { + summs[i], summs[j] = summs[j], summs[i] +} + +// sort interface implementation for summarydatasets +func (summs ScoreSummaryComparator) Less(i, j int) bool { + return summs[i].Test.TestContent.TestName < summs[j].Test.TestContent.TestName +} + +// reporting object for student participation +type ParticipationDataSet struct { + Student xml.RegistrationRecord + School xml.SchoolInfo + EventInfos []EventInfo + Summary Summary +} + +type Summary map[string]string + +// helper type for test/event info +type EventInfo struct { + Event xml.NAPEvent + Test xml.NAPTest +} + +// make school id and name a type for transmission +type SchoolDetails struct { + ACARAId string + SchoolName string +} + +// summary object from codeframe +type CodeFrameDataSet struct { + Test xml.NAPTest + Testlet xml.NAPTestlet + Item xml.NAPTestItem +} + +// +// helper method that walks the structure to find location +// +func (cfd CodeFrameDataSet) GetItemLocationInTestlet(itemrefid string) string { + + // check if the sequence no. is known + for _, item := range cfd.Testlet.TestItemList.TestItem { + if item.TestItemRefId == itemrefid { + return item.SequenceNumber + } + } + + // if not see if the item is an alternative + // get the alternative list of items + // see if they have a sequence number in the testlet + for _, altItem := range cfd.Item.TestItemContent.ItemSubstitutedForList.SubstituteItem { + for _, item := range cfd.Testlet.TestItemList.TestItem { + if item.TestItemRefId == altItem.SubstituteItemRefId { + return item.SequenceNumber + } + } + } + + return "unknown" +} + +// +// helpers for deeply nested writing rubric details +// +func (cfd CodeFrameDataSet) GetWritingRubricDescriptor(rubrictype string) string { + + for _, rubric := range cfd.Item.TestItemContent.NAPWritingRubricList.NAPWritingRubric { + if strings.EqualFold(rubric.RubricType, rubrictype) { + return rubric.Descriptor + } + } + + return "unknown" +} + +// +// helpers for deeply nested writing rubric details +// +func (cfd CodeFrameDataSet) GetWritingRubricMax(rubrictype string) string { + + for _, rubric := range cfd.Item.TestItemContent.NAPWritingRubricList.NAPWritingRubric { + if strings.EqualFold(rubric.RubricType, rubrictype) { + return rubric.ScoreList.Score[0].MaxScoreValue + } + } + + return "unknown" + +} + +// +// +// +// +// diff --git a/naprr/reportbuilder.go b/naprr/reportbuilder.go new file mode 100644 index 0000000..d67b433 --- /dev/null +++ b/naprr/reportbuilder.go @@ -0,0 +1,61 @@ +package naprr + +import ( + "log" + "sync" +) + +// var sr = NewStreamReader() +// var rg = NewReportGenerator() + +type ReportBuilder struct { + sr *StreamReader + rg *ReportGenerator +} + +func NewReportBuilder() *ReportBuilder { + return &ReportBuilder{sr: NewStreamReader(), rg: NewReportGenerator()} +} + +func (rb *ReportBuilder) Run() { + + var wg sync.WaitGroup + + schools := rb.sr.GetSchoolDetails() + nd := rb.sr.GetNAPLANData() + + for _, subslice := range schools { + for _, school := range subslice { + wg.Add(1) + go rb.createSchoolReports(nd, school.ACARAId, &wg) + } + } + + wg.Add(1) + go rb.createTestReports(nd, &wg) + + // block until all reports generated + wg.Wait() + log.Println("All reports generated") + +} + +// generate school-level data reports +func (rb *ReportBuilder) createSchoolReports(nd *NAPLANData, acaraid string, wg *sync.WaitGroup) { + sd := rb.sr.GetSchoolData(acaraid) + rb.rg.GenerateParticipationData(nd, sd) + log.Println("Participation data created for: ", acaraid) + rb.rg.GenerateSchoolScoreSummaryData(nd, sd) + log.Println("Score summary data created for: ", acaraid) + rb.rg.GenerateDomainScoreData(nd, sd) + log.Println("Domain scores data created for: ", acaraid) + + wg.Done() +} + +// generate test-level reports +func (rb *ReportBuilder) createTestReports(nd *NAPLANData, wg *sync.WaitGroup) { + rb.rg.GenerateCodeFrameData(nd) + log.Println("Codeframe data created.") + wg.Done() +} diff --git a/naprr/reportgenerator.go b/naprr/reportgenerator.go new file mode 100644 index 0000000..97fb97b --- /dev/null +++ b/naprr/reportgenerator.go @@ -0,0 +1,189 @@ +// reportgenerator.go +package naprr + +import ( + "github.com/nats-io/go-nats-streaming" + "github.com/nsip/nias2/lib" + "log" +) + +type ReportGenerator struct { + sc stan.Conn + ge GobEncoder +} + +func NewReportGenerator() *ReportGenerator { + rg := ReportGenerator{ + sc: CreateSTANConnection(), + ge: GobEncoder{}, + } + return &rg +} + +// +// routines to build the required reports +// + +// generate codeframe objects (currently as per VCAA requirements) +// generated only once as represents strucure of test not school-level data +func (rg *ReportGenerator) GenerateCodeFrameData(nd *NAPLANData) { + + count := 0 + cfds := make([]CodeFrameDataSet, 0) + + for _, codeframe := range nd.Codeframes { + for _, cf_testlet := range codeframe.TestletList.Testlet { + tl := nd.Testlets[cf_testlet.NAPTestletRefId] + // log.Printf("\t%s", tl.TestletContent.TestletName) + for _, cf_item := range cf_testlet.TestItemList.TestItem { + ti := nd.Items[cf_item.TestItemRefId] + // log.Printf("\t\t%s", ti.TestItemContent.ItemName) + cfd := CodeFrameDataSet{ + Test: nd.Tests[codeframe.NAPTestRefId], + Testlet: tl, + Item: ti, + } + cfds = append(cfds, cfd) + } + } + } + + count = len(cfds) + + // publish the records + for _, cfd := range cfds { + payload, err := rg.ge.Encode(cfd) + if err != nil { + log.Println("unable to encode codeframe: ", err) + } + // log.Printf("\t%s - %s - %s", cfd.Test.TestContent.TestDomain, + // cfd.Testlet.TestletContent.TestletName, cfd.Item.TestItemContent.ItemName) + rg.sc.Publish("reports.cframe", payload) + } + + // finish the transaction - completion msg + txu := lib.TxStatusUpdate{TxComplete: true} + gtxu, err := rg.ge.Encode(txu) + if err != nil { + log.Println("unable to encode txu codeframe report: ", err) + } + rg.sc.Publish("reports.cframe", gtxu) + + log.Printf("codeframe records %d: ", count) + +} + +// generate domain score objects +func (rg *ReportGenerator) GenerateDomainScoreData(nd *NAPLANData, sd *SchoolData) { + + count := 0 + + for _, response := range sd.Responses { + rds := ResponseDataSet{ + Test: nd.Tests[response.TestID], + Response: response, + } + // log.Printf("sc_score_summ:\n\n%v\n\n%v\n\n", rds.Test, rds.Response) + + payload, err := rg.ge.Encode(rds) + if err != nil { + log.Println("unable to encode domain scores: ", err) + } + rg.sc.Publish("reports."+sd.ACARAId+".dscores", payload) + + count++ + } + + // finish the transaction - completion msg + txu := lib.TxStatusUpdate{TxComplete: true} + gtxu, err := rg.ge.Encode(txu) + if err != nil { + log.Println("unable to encode txu domain scores report: ", err) + } + rg.sc.Publish("reports."+sd.ACARAId+".dscores", gtxu) + + log.Printf("domain score records %d: ", count) + +} + +// generate school summary objects +func (rg *ReportGenerator) GenerateSchoolScoreSummaryData(nd *NAPLANData, sd *SchoolData) { + + count := 0 + + for _, scoresummary := range sd.ScoreSummaries { + scsumm := ScoreSummaryDataSet{ + Test: nd.Tests[scoresummary.NAPTestRefId], + Summ: scoresummary, + } + // log.Printf("sc_score_summ:\n\n%v\n\n%v\n\n", scsumm.Test, scsumm.Summ) + + payload, err := rg.ge.Encode(scsumm) + if err != nil { + log.Println("unable to encode sch. summ: ", err) + } + rg.sc.Publish("reports."+sd.ACARAId+".scsumm", payload) + + count++ + } + + // finish the transaction - completion msg + txu := lib.TxStatusUpdate{TxComplete: true} + gtxu, err := rg.ge.Encode(txu) + if err != nil { + log.Println("unable to encode txu score summary report: ", err) + } + rg.sc.Publish("reports."+sd.ACARAId+".scsumm", gtxu) + + log.Printf("score summary records %d: ", count) + +} + +// generate the participation summary objects +func (rg *ReportGenerator) GenerateParticipationData(nd *NAPLANData, sd *SchoolData) { + + count := 0 + studentEvents := make(map[string][]EventInfo) + + for _, event := range sd.Events { + ei := EventInfo{Event: event, Test: nd.Tests[event.TestID]} + infos, ok := studentEvents[event.SPRefID] + if !ok { + infos = make([]EventInfo, 0) + } + infos = append(infos, ei) + studentEvents[event.SPRefID] = infos + } + + for _, student := range sd.Students { + pds := ParticipationDataSet{ + Student: sd.Students[student.RefId], + School: sd.SchoolInfos[student.ASLSchoolId], + EventInfos: studentEvents[student.RefId], + Summary: make(map[string]string), + } + for _, ei := range pds.EventInfos { + pds.Summary[ei.Test.TestContent.TestDomain] = ei.Event.ParticipationCode + } + payload, err := rg.ge.Encode(pds) + if err != nil { + log.Println("unable to encode pds: ", err) + } + rg.sc.Publish("reports."+sd.ACARAId+".particip", payload) + + count++ + } + + // finish the transaction - completion msg + txu := lib.TxStatusUpdate{TxComplete: true} + gtxu, err := rg.ge.Encode(txu) + if err != nil { + log.Println("unable to encode txu particip. report: ", err) + } + rg.sc.Publish("reports."+sd.ACARAId+".particip", gtxu) + + log.Printf("particpation records %d: ", count) +} + +// +// diff --git a/naprr/reportwriter.go b/naprr/reportwriter.go new file mode 100644 index 0000000..c6e37c1 --- /dev/null +++ b/naprr/reportwriter.go @@ -0,0 +1,501 @@ +package naprr + +import ( + "bufio" + "bytes" + "fmt" + "io" + "log" + "os" + "sync" + "text/template" +) + +// var rw_sr = NewStreamReader() +// var t *template.Template + +type ReportWriter struct { + sr *StreamReader + t *template.Template +} + +func NewReportWriter() *ReportWriter { + return &ReportWriter{sr: NewStreamReader(), t: loadTemplates()} +} + +func (rw *ReportWriter) Run() { + + // loadTemplates() + + schools := rw.sr.GetSchoolDetails() + + rw.writeSchoolLevelReports(schools) + rw.writeAggregateSchoolReports(schools) + rw.writeTestLevelReports() + + log.Println("All reports written\n") + +} + +// create data reports from the test strucutre +func (rw *ReportWriter) writeTestLevelReports() { + + log.Println("Creating test-level reports...") + + var wg sync.WaitGroup + + cfds := rw.sr.GetCodeFrameData() + + wg.Add(2) + + go rw.writeCodeFrameReport(cfds, &wg) + go rw.writeCodeFrameWritingReport(cfds, &wg) + + wg.Wait() + + log.Println("Test-level reports created.") +} + +// create data reports for each school +func (rw *ReportWriter) writeSchoolLevelReports(schools [][]SchoolDetails) { + + var wg sync.WaitGroup + + log.Println("Creating school-level reports...") + + for _, subslice := range schools { + for _, school := range subslice { + wg.Add(1) + go rw.writeSchoolReports(school.ACARAId, &wg) + } + } + + wg.Wait() + + log.Println("School-level reports created.") +} + +// create aggregate reports from school-level data +// uses file-concat for speed and to manage no. open connections & filehandles +// esp. on eg win32 environment +func (rw *ReportWriter) writeAggregateSchoolReports(schools [][]SchoolDetails) { + + log.Println("Creating aggregate reports...") + + outputPath := "out/" + + //report types we want to aggregate + reportTypes := []string{"participation", "score_summary", "domain_scores"} + + for _, reportType := range reportTypes { + // create empty aggregate report file with header + outputFile := rw.createSummaryFileWithHeader(reportType) + for _, subslice := range schools { + filePaths := make([]string, 0) + for _, schoolDetails := range subslice { + filePath := outputPath + schoolDetails.ACARAId + "/" + reportType + ".dat" + // check whether the file exists, ignore if doesn't + _, err := os.Stat(filePath) + if err != nil { + continue + } + filePaths = append(filePaths, filePath) + } + if len(filePaths) > 0 { + concatenateFiles(filePaths, outputFile) + // rewmove temp data files + for _, file := range filePaths { + err := os.Remove(file) + if err != nil { + fmt.Println("Unable to remove temp data file: ", file, err) + } + } + } + } + } + + log.Println("Aggregate reports created.") + +} + +// load all output templates once at start-up +func loadTemplates() *template.Template { + + t := template.Must(template.ParseGlob("templates/*")) + // log.Println(t.DefinedTemplates()) + return t +} + +func (rw *ReportWriter) createSummaryFileWithHeader(reportType string) (filePath string) { + + fname := "out/" + reportType + ".csv" + + var tmpl *template.Template + switch reportType { + case "participation": + tmpl = rw.t.Lookup("participation_hdr.tmpl") + case "score_summary": + tmpl = rw.t.Lookup("score_summary_hdr.tmpl") + case "domain_scores": + tmpl = rw.t.Lookup("domainscore_hdr.tmpl") + } + + // remove any previous versions + err := os.RemoveAll(fname) + if err != nil { + fmt.Println("Cannot delete previous aggregate file: ", fname) + } + + aggregateFile, err := os.Create(fname) + defer aggregateFile.Close() + if err != nil { + fmt.Println("Cannot open aggregate file: ", fname, err) + } + + // write the header + // doesn't actually need any data - all text fields so pass nil struct as data + if err := tmpl.Execute(aggregateFile, nil); err != nil { + fmt.Println("Cannot execute template header: ", reportType, err) + } + + aggregateFile.Close() + + return fname + +} + +func (rw *ReportWriter) writeSchoolReports(acaraid string, wg *sync.WaitGroup) { + + rw.writeParticipationReport(acaraid) + rw.writeScoreSummaryReport(acaraid) + rw.writeDomainScoreReport(acaraid) + + wg.Done() +} + +// report of test strucure for writing items only +// with extended item information +func (rw *ReportWriter) writeCodeFrameWritingReport(cfds []CodeFrameDataSet, wg *sync.WaitGroup) { + + thdr := rw.t.Lookup("codeframe_writing_hdr.tmpl") + trow := rw.t.Lookup("codeframe_writing_row.tmpl") + + // create directory for the school + fpath := "out/" + err := os.MkdirAll(fpath, os.ModePerm) + check(err) + + // create the report data file in the output directory + // delete any ecisting files and create empty new one + fname := fpath + "codeframe_writing.dat" + err = os.RemoveAll(fname) + f, err := os.Create(fname) + check(err) + defer f.Close() + + // write the data - writing items only + for _, cfd := range cfds { + if cfd.Test.TestContent.TestDomain == "Writing" { + if err := trow.Execute(f, cfd); err != nil { + check(err) + } + } + } + + // write the empty header file + fname2 := fpath + "codeframe_writing.csv" + f2, err := os.Create(fname2) + check(err) + defer f2.Close() + + // doesn't actually need any data - all text fields so pass nil struct as data + if err := thdr.Execute(f2, nil); err != nil { + check(err) + } + + inputFile := []string{fname} + outputFile := fname2 + + concatenateFiles(inputFile, outputFile) + + // remove the temp data files + err = os.RemoveAll(fname) + check(err) + + log.Printf("Codeframe writing report created for: %d elements", len(cfds)) + + wg.Done() + +} + +// report of test structure, is written only once +// as an aggrregate report, not at school level +func (rw *ReportWriter) writeCodeFrameReport(cfds []CodeFrameDataSet, wg *sync.WaitGroup) { + + thdr := rw.t.Lookup("codeframe_hdr.tmpl") + trow := rw.t.Lookup("codeframe_row.tmpl") + + // create directory for the school + fpath := "out/" + err := os.MkdirAll(fpath, os.ModePerm) + check(err) + + // create the report data file in the output directory + // delete any ecisting files and create empty new one + fname := fpath + "codeframe.dat" + err = os.RemoveAll(fname) + f, err := os.Create(fname) + check(err) + defer f.Close() + + // write the data - ignoring writing domain items, reported separately + for _, cfd := range cfds { + if !(cfd.Test.TestContent.TestDomain == "Writing") { + if err := trow.Execute(f, cfd); err != nil { + check(err) + } + + } + } + + // write the empty header file + fname2 := fpath + "codeframe.csv" + f2, err := os.Create(fname2) + check(err) + defer f2.Close() + + // doesn't actually need any data - all text fields so pass nil struct as data + if err := thdr.Execute(f2, nil); err != nil { + check(err) + } + + inputFile := []string{fname} + outputFile := fname2 + + concatenateFiles(inputFile, outputFile) + + // remove the temp data files + err = os.RemoveAll(fname) + check(err) + + log.Printf("Codeframe report created for: %d elements", len(cfds)) + + wg.Done() + +} + +func (rw *ReportWriter) writeDomainScoreReport(acaraid string) { + + thdr := rw.t.Lookup("domainscore_hdr.tmpl") + trow := rw.t.Lookup("domainscore_row.tmpl") + + // create directory for the school + fpath := "out/" + acaraid + err := os.MkdirAll(fpath, os.ModePerm) + check(err) + + // create the report data file in the directory + // delete any ecisting files and create empty new one + fname := fpath + "/domain_scores.dat" + err = os.RemoveAll(fname) + f, err := os.Create(fname) + check(err) + defer f.Close() + + // write the data + rds := rw.sr.GetDomainScoreData(acaraid) + for _, rd := range rds { + if err := trow.Execute(f, rd); err != nil { + check(err) + } + } + + // write the empty header file + fname2 := fpath + "/domain_scores.csv" + f2, err := os.Create(fname2) + check(err) + defer f2.Close() + + // doesn't actually need any data - all text fields so pass nil struct as data + if err := thdr.Execute(f2, nil); err != nil { + check(err) + } + + inputFile := []string{fname} + outputFile := fname2 + + concatenateFiles(inputFile, outputFile) + + log.Printf("Domain scores report created for: %s %d response-sets", acaraid, len(rds)) + +} + +func (rw *ReportWriter) writeParticipationReport(acaraid string) { + + thdr := rw.t.Lookup("participation_hdr.tmpl") + trow := rw.t.Lookup("participation_row.tmpl") + + // create directory for the school + fpath := "out/" + acaraid + err := os.MkdirAll(fpath, os.ModePerm) + check(err) + + // create the report data file in the directory + // delete any ecisting files and create empty new one + fname := fpath + "/participation.dat" + err = os.RemoveAll(fname) + f, err := os.Create(fname) + check(err) + defer f.Close() + + // write the data + pds := rw.sr.GetParticipationData(acaraid) + for _, pd := range pds { + if err := trow.Execute(f, pd); err != nil { + check(err) + } + } + + // write the empty header file + fname2 := fpath + "/participation.csv" + f2, err := os.Create(fname2) + check(err) + defer f2.Close() + + // doesn't actually need any data - all text fields so pass nil struct as data + if err := thdr.Execute(f2, nil); err != nil { + check(err) + } + + inputFile := []string{fname} + outputFile := fname2 + + concatenateFiles(inputFile, outputFile) + + log.Printf("Participation report created for: %s %d students", acaraid, len(pds)) + +} + +func (rw *ReportWriter) writeScoreSummaryReport(acaraid string) { + + thdr := rw.t.Lookup("score_summary_hdr.tmpl") + trow := rw.t.Lookup("score_summary_row.tmpl") + + // create directory for the school + fpath := "out/" + acaraid + err := os.MkdirAll(fpath, os.ModePerm) + check(err) + + // create the report data file in the directory + // delete any ecisting files and create empty new one + fname := fpath + "/score_summary.dat" + err = os.RemoveAll(fname) + f, err := os.Create(fname) + check(err) + defer f.Close() + + // write the data + ssds := rw.sr.GetScoreSummaryData(acaraid) + for _, ssd := range ssds { + if err := trow.Execute(f, ssd); err != nil { + check(err) + } + } + + // write the empty header file + fname2 := fpath + "/score_summary.csv" + f2, err := os.Create(fname2) + check(err) + defer f2.Close() + + // doesn't actually need any data - all text fields so pass nil struct as data + if err := thdr.Execute(f2, nil); err != nil { + check(err) + } + + inputFile := []string{fname} + outputFile := fname2 + + concatenateFiles(inputFile, outputFile) + + log.Printf("School score summary report created for: %s", acaraid) + +} + +// take a set of input files and create a single merged output file +func concatenateFiles(inputFiles []string, outputFile string) { + + reader, err := createReader(inputFiles) + if err != nil { + printAndHold(fmt.Sprintf("An error occurred during read: %s", err.Error())) + return + } + + writer, err := createWriter(outputFile) + if err != nil { + printAndHold(fmt.Sprintf("An error occurred during write: %s", err.Error())) + return + } + + err = pipe(reader, writer) + if err != nil { + printAndHold(fmt.Sprintf("An error occurred during pipe: %s", err.Error())) + } + +} + +func createReader(filePaths []string) (reader io.Reader, err error) { + readers := []io.Reader{} + for _, filePath := range filePaths { + inputFile, err := os.Open(filePath) + if err != nil { + return nil, err + } + readers = append(readers, inputFile) + // readers = append(readers, newLineReader()) + } + + return io.MultiReader(readers...), nil +} + +func createWriter(filePath string) (writer *bufio.Writer, err error) { + + // aggregate output file must be opened as append to + // maintain headers + outputFile, err := os.OpenFile(filePath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0660) + if err != nil { + return nil, err + } + + return bufio.NewWriter(outputFile), nil +} + +func pipe(reader io.Reader, writer *bufio.Writer) (err error) { + _, err = writer.ReadFrom(reader) + if err != nil { + return + } + + err = writer.Flush() + if err != nil { + return + } + + return +} + +func newLineReader() io.Reader { + newLine := []byte("\r\n") + return bytes.NewReader(newLine) +} + +func printAndHold(msg string) { + fmt.Println(msg) + fmt.Scan() +} + +func check(e error) { + if e != nil { + log.Println("Error writing report file: ", e) + } +} diff --git a/naprr/streamreader.go b/naprr/streamreader.go new file mode 100644 index 0000000..72054e2 --- /dev/null +++ b/naprr/streamreader.go @@ -0,0 +1,393 @@ +// streamreader.go +// +// +// utility routines to read naplan and school data from a +// stan stream + +package naprr + +import ( + "github.com/nats-io/go-nats-streaming" + "github.com/nsip/nias2/lib" + "github.com/nsip/nias2/xml" + "log" +) + +type StreamReader struct { + sc stan.Conn + ge GobEncoder +} + +func NewStreamReader() *StreamReader { + sr := StreamReader{ + sc: CreateSTANConnection(), + ge: GobEncoder{}, + } + return &sr +} + +func (sr *StreamReader) GetCodeFrameData() []CodeFrameDataSet { + + cfds := make([]CodeFrameDataSet, 0) + + // signal channel to notify asynch stan stream read is complete + txComplete := make(chan bool) + + // main message handling callback for the stan stream + // get names of schools that have been processed by ingest + // and create reports + mcb := func(m *stan.Msg) { + + // as we don't know message type ([]byte slice on wire) decode as interface + // then assert type dynamically + var m_if interface{} + err := sr.ge.Decode(m.Data, &m_if) + if err != nil { + log.Println("streamreader codeframe message decoding error: ", err) + txComplete <- true + } + + switch t := m_if.(type) { + case CodeFrameDataSet: + cfd := m_if.(CodeFrameDataSet) + cfds = append(cfds, cfd) + case lib.TxStatusUpdate: + txComplete <- true + default: + _ = t + // log.Printf("unknown message type in participation data handler: %v", m_if) + } + } + + sub, err := sr.sc.Subscribe("reports.cframe", mcb, stan.DeliverAllAvailable()) + defer sub.Unsubscribe() + if err != nil { + log.Println("streamreader: stan subsciption error get codeframe data: ", err) + } + + <-txComplete + + return cfds + +} + +// +// get domain scores +// +func (sr *StreamReader) GetDomainScoreData(acaraid string) []ResponseDataSet { + + rds := make([]ResponseDataSet, 0) + // signal channel to notify asynch stan stream read is complete + txComplete := make(chan bool) + + // main message handling callback for the stan stream + // get names of schools that have been processed by ingest + // and create reports + mcb := func(m *stan.Msg) { + + // as we don't know message type ([]byte slice on wire) decode as interface + // then assert type dynamically + var m_if interface{} + err := sr.ge.Decode(m.Data, &m_if) + if err != nil { + log.Println("streamreader domain score message decoding error: ", err) + txComplete <- true + } + + switch t := m_if.(type) { + case ResponseDataSet: + rd := m_if.(ResponseDataSet) + rds = append(rds, rd) + case lib.TxStatusUpdate: + txComplete <- true + default: + _ = t + // log.Printf("unknown message type in participation data handler: %v", m_if) + } + } + + sub, err := sr.sc.Subscribe("reports."+acaraid+".dscores", mcb, stan.DeliverAllAvailable()) + defer sub.Unsubscribe() + if err != nil { + log.Println("streamreader: stan subsciption error get domain scores data: ", err) + } + + <-txComplete + + return rds + +} + +// +// get school score summaries +// +func (sr *StreamReader) GetScoreSummaryData(acaraid string) []ScoreSummaryDataSet { + + ssds := make([]ScoreSummaryDataSet, 0) + + // signal channel to notify asynch stan stream read is complete + txComplete := make(chan bool) + + // main message handling callback for the stan stream + // get names of schools that have been processed by ingest + // and create reports + mcb := func(m *stan.Msg) { + + // as we don't know message type ([]byte slice on wire) decode as interface + // then assert type dynamically + var m_if interface{} + err := sr.ge.Decode(m.Data, &m_if) + if err != nil { + log.Println("streamreader score summ message decoding error: ", err) + txComplete <- true + } + + switch t := m_if.(type) { + case ScoreSummaryDataSet: + ssd := m_if.(ScoreSummaryDataSet) + ssds = append(ssds, ssd) + case lib.TxStatusUpdate: + txComplete <- true + default: + _ = t + // log.Printf("unknown message type in participation data handler: %v", m_if) + } + } + + sub, err := sr.sc.Subscribe("reports."+acaraid+".scsumm", mcb, stan.DeliverAllAvailable()) + defer sub.Unsubscribe() + if err != nil { + log.Println("streamreader: stan subsciption error get score summary data: ", err) + } + + <-txComplete + + return ssds + +} + +// +// retrieve participation data for the given school +// +func (sr *StreamReader) GetParticipationData(acaraid string) []ParticipationDataSet { + + pds := make([]ParticipationDataSet, 0) + + // signal channel to notify asynch stan stream read is complete + txComplete := make(chan bool) + + // main message handling callback for the stan stream + // get names of schools that have been processed by ingest + // and create reports + mcb := func(m *stan.Msg) { + + // as we don't know message type ([]byte slice on wire) decode as interface + // then assert type dynamically + var m_if interface{} + err := sr.ge.Decode(m.Data, &m_if) + if err != nil { + log.Println("streamreader participation message decoding error: ", err) + txComplete <- true + } + + switch t := m_if.(type) { + case ParticipationDataSet: + pd := m_if.(ParticipationDataSet) + pds = append(pds, pd) + case lib.TxStatusUpdate: + txComplete <- true + default: + _ = t + // log.Printf("unknown message type in participation data handler: %v", m_if) + } + } + + sub, err := sr.sc.Subscribe("reports."+acaraid+".particip", mcb, stan.DeliverAllAvailable()) + defer sub.Unsubscribe() + if err != nil { + log.Println("streamreader: stan subsciption error get participation data: ", err) + } + + <-txComplete + + return pds + +} + +// returns simple list of all schools that have been processed +// from the results data set +// +// details are a slice of slices so that downstream processes +// such as concurrent file writes can work with a moderate batch size +// e.g. 100 at a time to prevent issues with too many open filehandles +// stan connections etc. +func (sr *StreamReader) GetSchoolDetails() [][]SchoolDetails { + + sds := make([]SchoolDetails, 0) + + // signal channel to notify asynch stan stream read is complete + txComplete := make(chan bool) + + // main message handling callback for the stan stream + // get names of schools that have been processed by ingest + // and create reports + mcb := func(m *stan.Msg) { + + // as we don't know message type ([]byte slice on wire) decode as interface + // then assert type dynamically + var m_if interface{} + err := sr.ge.Decode(m.Data, &m_if) + if err != nil { + log.Println("message decoding error: ", err) + txComplete <- true + } + + switch t := m_if.(type) { + case SchoolDetails: + sd := m_if.(SchoolDetails) + sds = append(sds, sd) + case lib.TxStatusUpdate: + txComplete <- true + default: + _ = t + // log.Printf("unknown message type in stream reader school details handler: %v", m_if) + } + } + + sub, err := sr.sc.Subscribe("schools", mcb, stan.DeliverAllAvailable()) + defer sub.Unsubscribe() + if err != nil { + log.Println("streamreader: stan subsciption error get school details: ", err) + } + + <-txComplete + + // now chunk up the list of schools into sub-slices + var sds_chunks [][]SchoolDetails + chunkSize := 99 + + for i := 0; i < len(sds); i += chunkSize { + end := i + chunkSize + + if end > len(sds) { + end = len(sds) + } + + sds_chunks = append(sds_chunks, sds[i:end]) + } + + return sds_chunks + +} + +// NAPLAN data is the same for all schools, so can be retrieved once +func (sr *StreamReader) GetNAPLANData() *NAPLANData { + + nd := NewNAPLANData() + + // signal channel to notify asynch stan stream read is complete + txComplete := make(chan bool) + + // main message handling callback for the stan stream + mcb := func(m *stan.Msg) { + + // as we don't know message type ([]byte slice on wire) decode as interface + // then assert type dynamically + var m_if interface{} + err := sr.ge.Decode(m.Data, &m_if) + if err != nil { + log.Println("streamreader: schooldata message decoding error: ", err) + txComplete <- true + } + + switch mtype := m_if.(type) { + case xml.NAPTest: + t := m_if.(xml.NAPTest) + nd.Tests[t.TestID] = t + case xml.NAPTestlet: + tl := m_if.(xml.NAPTestlet) + nd.Testlets[tl.TestletID] = tl + case xml.NAPTestItem: + ti := m_if.(xml.NAPTestItem) + nd.Items[ti.ItemID] = ti + case xml.NAPCodeFrame: + cf := m_if.(xml.NAPCodeFrame) + nd.Codeframes[cf.NAPTestRefId] = cf + case lib.TxStatusUpdate: + txComplete <- true + default: + _ = mtype + // log.Printf("unknown message type in stream reader meta handler: %v", m_if) + } + + } + + sub, err := sr.sc.Subscribe("meta", mcb, stan.DeliverAllAvailable()) + defer sub.Unsubscribe() + if err != nil { + log.Println("streamreader: stan subsciption error meta channel: ", err) + } + + <-txComplete + + return nd + +} + +// for the school identified by the acaraid retrieves all of the +// raw results data objects +func (sr *StreamReader) GetSchoolData(acaraid string) *SchoolData { + + sd := NewSchoolData(acaraid) + + // signal channel to notify asynch stan stream read is complete + txComplete := make(chan bool) + + // main message handling callback for the stan stream + mcb := func(m *stan.Msg) { + + // as we don't know message type ([]byte slice on wire) decode as interface + // then assert type dynamically + var m_if interface{} + err := sr.ge.Decode(m.Data, &m_if) + if err != nil { + log.Println("message decoding error: ", err) + txComplete <- true + } + + switch mtype := m_if.(type) { + case xml.SchoolInfo: + si := m_if.(xml.SchoolInfo) + sd.SchoolInfos[si.ACARAId] = si + case xml.NAPEvent: + e := m_if.(xml.NAPEvent) + sd.Events[e.EventID] = e + case xml.RegistrationRecord: + sp := m_if.(xml.RegistrationRecord) + sd.Students[sp.RefId] = sp + case xml.NAPTestScoreSummary: + tss := m_if.(xml.NAPTestScoreSummary) + sd.ScoreSummaries[tss.SummaryID] = tss + case xml.NAPResponseSet: + rs := m_if.(xml.NAPResponseSet) + sd.Responses[rs.ResponseID] = rs + case lib.TxStatusUpdate: + txComplete <- true + default: + _ = mtype + // log.Printf("unknown message type in stream reader schooldata handler: %v", m_if) + } + + } + + sub, err := sr.sc.Subscribe(sd.ACARAId, mcb, stan.DeliverAllAvailable()) + if err != nil { + log.Println("streamreader: stan subsciption error school data channel: ", err) + } + defer sub.Unsubscribe() + + <-txComplete + + return sd + +} diff --git a/lib/config.go b/napval/validation_config.go similarity index 98% rename from lib/config.go rename to napval/validation_config.go index 4d8e189..7298693 100644 --- a/lib/config.go +++ b/napval/validation_config.go @@ -1,5 +1,5 @@ // configmanager.go -package lib +package napval import ( "github.com/BurntSushi/toml" diff --git a/napval/validation_store.go b/napval/validation_store.go index 1d62b8a..6afc932 100644 --- a/napval/validation_store.go +++ b/napval/validation_store.go @@ -11,6 +11,8 @@ import ( // amount of error reports to store for any given input file var STORE_LIMIT = config.TxStorageLimit +//var STORE_LIMIT = DefaultValidationConfig.TxStorageLimit + // ValidationStore assigns messages (validation results) to // output streams for retrieval by clients type ValidationStore struct { diff --git a/release_naprr.sh b/release_naprr.sh new file mode 100644 index 0000000..7f4e17f --- /dev/null +++ b/release_naprr.sh @@ -0,0 +1,7 @@ + +cd tools; go build release.go; cd .. +./tools/release nias2 nias-naprr-Mac.zip build/nias-naprr-Mac.zip +./tools/release nias2 nias-naprr-Win64.zip build/nias-naprr-Win64.zip +./tools/release nias2 nias-naprr-Win32.zip build/nias-naprr-Win32.zip +./tools/release nias2 nias-naprr-Linux64.zip build/nias-naprr-Linux64.zip +./tools/release nias2 nias-naprr-Linux32.zip build/nias-naprr-Linux32.zip diff --git a/release.sh b/release_napval.sh similarity index 100% rename from release.sh rename to release_napval.sh diff --git a/test_data/100students.csv b/test_data/napval/100students.csv similarity index 100% rename from test_data/100students.csv rename to test_data/napval/100students.csv diff --git a/test_data/1500students.csv b/test_data/napval/1500students.csv similarity index 100% rename from test_data/1500students.csv rename to test_data/napval/1500students.csv diff --git a/test_data/1student.csv b/test_data/napval/1student.csv similarity index 100% rename from test_data/1student.csv rename to test_data/napval/1student.csv diff --git a/test_data/20students.csv b/test_data/napval/20students.csv similarity index 100% rename from test_data/20students.csv rename to test_data/napval/20students.csv diff --git a/test_data/5students.csv b/test_data/napval/5students.csv similarity index 100% rename from test_data/5students.csv rename to test_data/napval/5students.csv diff --git a/xml/naptest.go b/xml/naptest.go index c8e794f..04434c2 100644 --- a/xml/naptest.go +++ b/xml/naptest.go @@ -9,6 +9,7 @@ type NAPTest struct { TestDomain string `xml:"Domain"` TestYear string `xml:"TestYear"` StagesCount string `xml:"StagesCount"` + TestType string `xml:"TestType"` DomainBands struct { Band1Lower string `xml:"Band1Lower"` Band1Upper string `xml:"Band1Upper"` diff --git a/xml/naptestitem.go b/xml/naptestitem.go index 14be4c0..a18067f 100644 --- a/xml/naptestitem.go +++ b/xml/naptestitem.go @@ -51,6 +51,7 @@ type NAPTestItem struct { NAPWritingRubricList struct { NAPWritingRubric []struct { RubricType string `xml:"RubricType"` + Descriptor string `xml:"Descriptor"` ScoreList struct { Score []struct { MaxScoreValue string `xml:"MaxScoreValue"` @@ -62,7 +63,6 @@ type NAPTestItem struct { } `xml:"ScoreDescriptionList"` } `xml:"Score"` } `xml:"ScoreList"` - Descriptor string `xml:"Descriptor"` } `xml:"NAPWritingRubric"` } `xml:"NAPWritingRubricList"` } `xml:"TestItemContent"` diff --git a/xml/registrationrecord.go b/xml/registrationrecord.go index 4776fb8..e5f3fb1 100644 --- a/xml/registrationrecord.go +++ b/xml/registrationrecord.go @@ -4,10 +4,8 @@ import ( "encoding/gob" "encoding/xml" "github.com/nsip/nias2/go_SifMessage" - //"log" ) -// ensures transmissable types are registered for binary encoding func init() { // make gob encoder aware of local types gob.Register(RegistrationRecord{}) @@ -201,6 +199,18 @@ func (r *RegistrationRecord) Unflatten() RegistrationRecord { return *r } +// convenience method to return otherid by type +func (r RegistrationRecord) GetOtherId(idtype string) string { + + for _, id := range r.OtherIdList.OtherId { + if strings.EqualFold(id.Type, idtype) { + return id.Value + } + } + + return idtype +} + // convenience method for writing to csv func (r RegistrationRecord) GetHeaders() []string { return []string{"ASLSchoolId",