Trimming CSV fields on import

nsip · Dec 8, 2016 · 2b951d3 · 2b951d3
1 parent 9bb766e
commit 2b951d3
Show file tree

Hide file tree

Showing 8 changed files with 33 additions and 15 deletions.
diff --git a/harness/public/index.html b/harness/public/index.html
@@ -75,7 +75,7 @@ <h6 class="docs-header">How to use the pre-validation tool</h6>
                     <p>Select a csv file containing student registrations data with the file chooser on the left, you can drag files onto the highlighted area or browse for them.</p>
                     <p>The file will be uploaded and processed by all validation engines simultaneously. The Progress area on the right will show the progress of the validation activity. </p>
                     <p>Once all validations are complete the results will be displayed automatically in the graphs and tables below. </p>
-                    <p>If there are a very large number of validation errors, only the first 10,000 wil be returned to this web interface. </p>
+                    <p>If there are a very large number of validation errors, only the first 10,000 will be returned to this web interface. </p>
                     <p>Click on the 'Download Validation Error Report' button to download the validation results as an excel/csv file. This will include all validation errors and is not capped at 10,000.</p>
                 </div>
             </div>

diff --git a/harness/schemas/core.json b/harness/schemas/core.json
@@ -85,7 +85,6 @@
             "id": "http://naplan.edu.au/FTE",
             "type": "string",
             "title": "FTE schema.",
-            "maxLength": 4,
             "maximum": 1,
             "minimum": 0,
             "description": "Description",

diff --git a/harness/validator_test.go b/harness/validator_test.go
@@ -30,6 +30,10 @@ func TestYearLevelF(t *testing.T) {
 	test_harness(t, "../unit_test_files/1students2YearLevelF.csv", "BirthDate/YearLevel", "Student Year Level (yr F) does not match year level derived from BirthDate")
 }
 
+func TestYearLevelP(t *testing.T) {
+	test_harness(t, "../unit_test_files/1students1YearLevelP.csv", "BirthDate/YearLevel", "")
+}
+
 func TestFutureBirthdate(t *testing.T) {
 	test_harness(t, "../unit_test_files/1studentsFutureBirthDates.csv", "BirthDate/YearLevel", "Year Level calculated from BirthDate does not fall within expected NAPLAN year level ranges")
 }
@@ -278,6 +282,14 @@ func TestYearLevelTestLevelMismatch(t *testing.T) {
 	test_harness(t, "../unit_test_files/1studentsYearLevelTestLevelMismatch.csv", "BirthDate/TestLevel", "does not match year level derived from BirthDate")
 }
 
+func TestMaximumFTE(t *testing.T) {
+	test_harness(t, "../unit_test_files/1MaximumFTE.csv", "FTE", "Year level supplied is UG, will result in SRM warning flag for test level")
+}
+
+func TestMinimumFTE(t *testing.T) {
+	test_harness(t, "../unit_test_files/1MinimumFTE.csv", "FTE", "Year level supplied is UG, will result in SRM warning flag for test level")
+}
+
 func TestDupGivenLastNameDOBCARAId(t *testing.T) {
 	for i := 0; i < 20; i++ {
 		test_harness(t, "../unit_test_files/2studentsDupGivenLastNameDOBSchool.csv", "Multiple (see description)", "otential duplicate of record")

diff --git a/lib/vtypes.go b/lib/vtypes.go
@@ -18,15 +18,16 @@ type RegistrationRecord struct {
 	// XML Configuration
 	// XMLName            xml.Name `xml:"StudentPersonal"`
 	// Important fields
-	ASLSchoolId               string `json:",omitempty" xml:"MostRecent>SchoolACARAId"`
-	AddressLine1              string `json:",omitempty" xml:"PersonInfo>AddressList>Address>Street>Line1"`
-	AddressLine2              string `json:",omitempty" xml:"PersonInfo>AddressList>Address>Street>Line2"`
-	BirthDate                 string `json:",omitempty" xml:"PersonInfo>Demographics>BirthDate"`
-	ClassGroup                string `json:",omitempty" xml:"MostRecent>ClassCode"`
-	CountryOfBirth            string `json:",omitempty" xml:"PersonInfo>Demographics>CountryOfBirth"`
-	DiocesanId                string `json:",omitempty"`
-	EducationSupport          string `json:",omitempty" xml:"EducationSupport"`
-	FFPOS                     string `json:",omitempty" xml:"MostRecent>FFPOS"`
+	ASLSchoolId      string `json:",omitempty" xml:"MostRecent>SchoolACARAId"`
+	AddressLine1     string `json:",omitempty" xml:"PersonInfo>AddressList>Address>Street>Line1"`
+	AddressLine2     string `json:",omitempty" xml:"PersonInfo>AddressList>Address>Street>Line2"`
+	BirthDate        string `json:",omitempty" xml:"PersonInfo>Demographics>BirthDate"`
+	ClassGroup       string `json:",omitempty" xml:"MostRecent>ClassCode"`
+	CountryOfBirth   string `json:",omitempty" xml:"PersonInfo>Demographics>CountryOfBirth"`
+	DiocesanId       string `json:",omitempty"`
+	EducationSupport string `json:",omitempty" xml:"EducationSupport"`
+	FFPOS            string `json:",omitempty" xml:"MostRecent>FFPOS"`
+	//FTE                       float32 `json:",omitempty" xml:"MostRecent>FTE"`
 	FTE                       string `json:",omitempty" xml:"MostRecent>FTE"`
 	FamilyName                string `json:",omitempty" xml:"PersonInfo>Name>FamilyName"`
 	GivenName                 string `json:",omitempty" xml:"PersonInfo>Name>GivenName"`

diff --git a/lib/webserver.go b/lib/webserver.go
@@ -59,7 +59,7 @@ func removeBlanks(m map[string]string) map[string]string {
 	reducedmap := make(map[string]string)
 	for key, val := range m {
 		if val != "" {
-			reducedmap[key] = val
+			reducedmap[key] = strings.TrimSpace(val)
 		}
 	}
 	return reducedmap
@@ -383,9 +383,9 @@ func (nws *NIASWebServer) Run() {
 		sprsnls := make([]map[string]string, 0)
 		for _, r := range records {
 			r := r.AsMap()
-			r = removeBlanks(r)
-			r["SIFuuid"] = uuid.NewV4().String()
-			sprsnls = append(sprsnls, r)
+			r1 := removeBlanks(r)
+			r1["SIFuuid"] = uuid.NewV4().String()
+			sprsnls = append(sprsnls, r1)
 		}
 
 		// set headers to 'force' file download where appropriate

diff --git a/unit_test_files/1MaximumFTE.csv b/unit_test_files/1MaximumFTE.csv
@@ -0,0 +1,2 @@
+LocalId,SectorId,DiocesanId,OtherId,TAAId,StateProvinceId,NationalId,PlatformId,PreviousLocalId,PreviousSectorId,PreviousDiocesanId,PreviousOtherId,PreviousTAAId,PreviousStateProvinceId,PreviousNationalId,PreviousPlatformId,FamilyName,GivenName,PreferredName,MiddleName,BirthDate,Sex,CountryOfBirth,EducationSupport,FFPOS,VisaCode,IndigenousStatus,LBOTE,StudentLOTE,YearLevel,TestLevel,FTE,Homegroup,ClassGroup,ASLSchoolId,SchoolLocalId,LocalCampusId,MainSchoolFlag,OtherSchoolId,ReportingSchoolId,HomeSchooledStudent,Sensitive,OfflineDelivery,Parent1SchoolEducation,Parent1NonSchoolEducation,Parent1Occupation,Parent1LOTE,Parent2SchoolEducation,Parent2NonSchoolEducation,Parent2Occupation,Parent2LOTE,AddressLine1,AddressLine2,Locality,Postcode,StateTerritory
+gqhvy519,75346,84945,55629,24653,77815,33999,R100000081K,3374,75911,44624,29026,83755,46201,38284,R100000081K,Baker,Cynthia,Cynthia,C,2008-08-11,2,1101,X,2,101,2,Y,1201,3,3,1.63,03,3E,44370,37298,1,1,tjtzi844,lxwnp178,Y,Y,N,4,7,4,1201,1,6,1,1201,"634 Frenchmen Street,",,CORNELIA CREEK,3622,VIC
diff --git a/unit_test_files/1MinimumFTE.csv b/unit_test_files/1MinimumFTE.csv
@@ -0,0 +1,2 @@
+LocalId,SectorId,DiocesanId,OtherId,TAAId,StateProvinceId,NationalId,PlatformId,PreviousLocalId,PreviousSectorId,PreviousDiocesanId,PreviousOtherId,PreviousTAAId,PreviousStateProvinceId,PreviousNationalId,PreviousPlatformId,FamilyName,GivenName,PreferredName,MiddleName,BirthDate,Sex,CountryOfBirth,EducationSupport,FFPOS,VisaCode,IndigenousStatus,LBOTE,StudentLOTE,YearLevel,TestLevel,FTE,Homegroup,ClassGroup,ASLSchoolId,SchoolLocalId,LocalCampusId,MainSchoolFlag,OtherSchoolId,ReportingSchoolId,HomeSchooledStudent,Sensitive,OfflineDelivery,Parent1SchoolEducation,Parent1NonSchoolEducation,Parent1Occupation,Parent1LOTE,Parent2SchoolEducation,Parent2NonSchoolEducation,Parent2Occupation,Parent2LOTE,AddressLine1,AddressLine2,Locality,Postcode,StateTerritory
+gqhvy518,75346,84945,55629,24653,77815,33999,R100000081K,3374,75911,44624,29026,83755,46201,38284,R100000081K,Baker,Cynthia,Cynthia,C,2008-08-11,2,1101,X,2,101,2,Y,1201,3,3,-0.63,03,3E,44370,37298,1,1,tjtzi844,lxwnp178,Y,Y,N,4,7,4,1201,1,6,1,1201,"634 Frenchmen Street,",,CORNELIA CREEK,3622,VIC
diff --git a/unit_test_files/1students1YearLevelP.csv b/unit_test_files/1students1YearLevelP.csv
@@ -0,0 +1,2 @@
+LocalId,SectorId,DiocesanId,OtherId,TAAId,StateProvinceId,NationalId,PlatformId,PreviousLocalId,PreviousSectorId,PreviousDiocesanId,PreviousOtherId,PreviousTAAId,PreviousStateProvinceId,PreviousNationalId,PreviousPlatformId,FamilyName,GivenName,PreferredName,MiddleName,BirthDate,Sex,CountryOfBirth,EducationSupport,FFPOS,VisaCode,IndigenousStatus,LBOTE,StudentLOTE,YearLevel,TestLevel,FTE,Homegroup,ClassGroup,ASLSchoolId,SchoolLocalId,LocalCampusId,MainSchoolFlag,OtherSchoolId,ReportingSchoolId,HomeSchooledStudent,Sensitive,OfflineDelivery,Parent1SchoolEducation,Parent1NonSchoolEducation,Parent1Occupation,Parent1LOTE,Parent2SchoolEducation,Parent2NonSchoolEducation,Parent2Occupation,Parent2LOTE,AddressLine1,AddressLine2,Locality,Postcode,StateTerritory
+gqhvy520,75346,84945,55629,24653,77815,33999,R100000003S,3374,75911,44624,29026,83755,46201,38284,R100000003S,Baker,Cynthia,Cynthia,C,2007-08-11,2,1101,X,2,101,2,Y,1201,P,3,0.63,3D,3E,44370,37298,1,1,tjtzi844,lxwnp178,Y,Y,N,4,7,4,1201,1,6,1,1201,"634 Frenchmen Street,",,CORNELIA CREEK,3622,VIC
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		LocalId,SectorId,DiocesanId,OtherId,TAAId,StateProvinceId,NationalId,PlatformId,PreviousLocalId,PreviousSectorId,PreviousDiocesanId,PreviousOtherId,PreviousTAAId,PreviousStateProvinceId,PreviousNationalId,PreviousPlatformId,FamilyName,GivenName,PreferredName,MiddleName,BirthDate,Sex,CountryOfBirth,EducationSupport,FFPOS,VisaCode,IndigenousStatus,LBOTE,StudentLOTE,YearLevel,TestLevel,FTE,Homegroup,ClassGroup,ASLSchoolId,SchoolLocalId,LocalCampusId,MainSchoolFlag,OtherSchoolId,ReportingSchoolId,HomeSchooledStudent,Sensitive,OfflineDelivery,Parent1SchoolEducation,Parent1NonSchoolEducation,Parent1Occupation,Parent1LOTE,Parent2SchoolEducation,Parent2NonSchoolEducation,Parent2Occupation,Parent2LOTE,AddressLine1,AddressLine2,Locality,Postcode,StateTerritory
		gqhvy519,75346,84945,55629,24653,77815,33999,R100000081K,3374,75911,44624,29026,83755,46201,38284,R100000081K,Baker,Cynthia,Cynthia,C,2008-08-11,2,1101,X,2,101,2,Y,1201,3,3,1.63,03,3E,44370,37298,1,1,tjtzi844,lxwnp178,Y,Y,N,4,7,4,1201,1,6,1,1201,"634 Frenchmen Street,",,CORNELIA CREEK,3622,VIC
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		LocalId,SectorId,DiocesanId,OtherId,TAAId,StateProvinceId,NationalId,PlatformId,PreviousLocalId,PreviousSectorId,PreviousDiocesanId,PreviousOtherId,PreviousTAAId,PreviousStateProvinceId,PreviousNationalId,PreviousPlatformId,FamilyName,GivenName,PreferredName,MiddleName,BirthDate,Sex,CountryOfBirth,EducationSupport,FFPOS,VisaCode,IndigenousStatus,LBOTE,StudentLOTE,YearLevel,TestLevel,FTE,Homegroup,ClassGroup,ASLSchoolId,SchoolLocalId,LocalCampusId,MainSchoolFlag,OtherSchoolId,ReportingSchoolId,HomeSchooledStudent,Sensitive,OfflineDelivery,Parent1SchoolEducation,Parent1NonSchoolEducation,Parent1Occupation,Parent1LOTE,Parent2SchoolEducation,Parent2NonSchoolEducation,Parent2Occupation,Parent2LOTE,AddressLine1,AddressLine2,Locality,Postcode,StateTerritory
		gqhvy518,75346,84945,55629,24653,77815,33999,R100000081K,3374,75911,44624,29026,83755,46201,38284,R100000081K,Baker,Cynthia,Cynthia,C,2008-08-11,2,1101,X,2,101,2,Y,1201,3,3,-0.63,03,3E,44370,37298,1,1,tjtzi844,lxwnp178,Y,Y,N,4,7,4,1201,1,6,1,1201,"634 Frenchmen Street,",,CORNELIA CREEK,3622,VIC
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		LocalId,SectorId,DiocesanId,OtherId,TAAId,StateProvinceId,NationalId,PlatformId,PreviousLocalId,PreviousSectorId,PreviousDiocesanId,PreviousOtherId,PreviousTAAId,PreviousStateProvinceId,PreviousNationalId,PreviousPlatformId,FamilyName,GivenName,PreferredName,MiddleName,BirthDate,Sex,CountryOfBirth,EducationSupport,FFPOS,VisaCode,IndigenousStatus,LBOTE,StudentLOTE,YearLevel,TestLevel,FTE,Homegroup,ClassGroup,ASLSchoolId,SchoolLocalId,LocalCampusId,MainSchoolFlag,OtherSchoolId,ReportingSchoolId,HomeSchooledStudent,Sensitive,OfflineDelivery,Parent1SchoolEducation,Parent1NonSchoolEducation,Parent1Occupation,Parent1LOTE,Parent2SchoolEducation,Parent2NonSchoolEducation,Parent2Occupation,Parent2LOTE,AddressLine1,AddressLine2,Locality,Postcode,StateTerritory
		gqhvy520,75346,84945,55629,24653,77815,33999,R100000003S,3374,75911,44624,29026,83755,46201,38284,R100000003S,Baker,Cynthia,Cynthia,C,2007-08-11,2,1101,X,2,101,2,Y,1201,P,3,0.63,3D,3E,44370,37298,1,1,tjtzi844,lxwnp178,Y,Y,N,4,7,4,1201,1,6,1,1201,"634 Frenchmen Street,",,CORNELIA CREEK,3622,VIC