Merge pull request #88 from MojioMS/feature/no-data-values

Feature/no data values
52North · Sep 1, 2017 · 6a0c419 · 6a0c419
2 parents 1b91456 + 0b8ed35
commit 6a0c419
Show file tree

Hide file tree

Showing 7 changed files with 186 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -247,6 +247,9 @@ Please take a look at the [github issues list](https://github.com/52North/sos-im
 ### :white_large_square: 0.5
 
    * *Features*
+     * Add support for no data values for time series in
+       * Bindings
+       * Feeder
      * Add support to ignore lines with invalid number of columns (=> log error but no import abort) in
        * Wizard
        * Bindings

diff --git a/bindings/src/main/resources/import-configuration.xsd b/bindings/src/main/resources/import-configuration.xsd
@@ -133,6 +133,13 @@
 						<xs:documentation>
                             Indicates the membership of this column in a 
                             POSITION or DATE_TIME group.
+                        </xs:documentation>
+					</xs:annotation>
+				</xs:enumeration>
+				<xs:enumeration value="NO_DATA_VALUE">
+					<xs:annotation>
+						<xs:documentation>
+                            Indicates a missing value of this column. <!--FIXME: more specify?-->
                         </xs:documentation>
 					</xs:annotation>
 				</xs:enumeration>

diff --git a/feeder/src/main/java/org/n52/sos/importer/feeder/Configuration.java b/feeder/src/main/java/org/n52/sos/importer/feeder/Configuration.java
@@ -122,6 +122,8 @@ public final class Configuration {
     public static final String SOS_EXCEPTION_OBSERVATION_DUPLICATE_CONSTRAINT = "observation_time_stamp_key";
     /** Constant <code>SOS_OBSERVATION_ALREADY_CONTAINED="observation already contained in sos"</code> */
     public static final String SOS_OBSERVATION_ALREADY_CONTAINED = "observation already contained in sos";
+    /** Constant <code>SOS_OBSERVATION_TYPE_NO_DATA_VALUE="NO_DATA_VALUE"</code> */
+    public static final String SOS_OBSERVATION_TYPE_NO_DATA_VALUE = "NO_DATA_VALUE";
     /** Constant <code>SOS_OBSERVATION_TYPE_TEXT="TEXT"</code> */
     public static final String SOS_OBSERVATION_TYPE_TEXT = "TEXT";
     /** Constant <code>SOS_OBSERVATION_TYPE_COUNT="COUNT"</code> */
@@ -1652,4 +1654,22 @@ public List<Column> getColumnsForOmParameter(int mVColumnId) {
         return Collections.emptyList();
     }
 
+    public boolean isNoDataValueDefinedAndMatching(Column column, String value) {
+        if (value == null || value.isEmpty()) {
+            return false;
+        }
+        if (column == null || column.getMetadataArray() == null || column.sizeOfMetadataArray() == 0) {
+            return false;
+        }
+        for (Metadata md : column.getMetadataArray()) {
+            if (md.getKey().equals(Key.NO_DATA_VALUE)) {
+                if (value.equals(md.getValue())) {
+                    LOG.trace("value '{}' is matching NO_DATA_VALUE '{}'.", value, md.getValue());
+                    return true;
+                }
+                return false;
+            }
+        }
+        return false;
+    }
 }
diff --git a/feeder/src/main/java/org/n52/sos/importer/feeder/DataFile.java b/feeder/src/main/java/org/n52/sos/importer/feeder/DataFile.java
@@ -431,6 +431,9 @@ public Object getValue(final int mVColumn, final String[] values) throws ParseEx
                 Arrays.toString(values)));
         final Column column = configuration.getColumnById(mVColumn);
         String value = values[mVColumn];
+        if (configuration.isNoDataValueDefinedAndMatching(column, value)) {
+                    return Configuration.SOS_OBSERVATION_TYPE_NO_DATA_VALUE;
+        }
         for (final Metadata m : column.getMetadataArray()) {
             if (m.getKey().equals(Key.TYPE)) {
                 // check various types of observation

diff --git a/feeder/src/main/java/org/n52/sos/importer/feeder/SensorObservationService.java b/feeder/src/main/java/org/n52/sos/importer/feeder/SensorObservationService.java
@@ -695,6 +695,9 @@ private InsertObservation getInsertObservationForColumnIdFromValues(final int mV
         LOG.debug("Feature of Interest: {}", foi);
         // VALUE
         final Object value = dataFile.getValue(mVColumnId, values);
+        if (value.equals(Configuration.SOS_OBSERVATION_TYPE_NO_DATA_VALUE)) {
+            return null;
+        }
         // TODO implement handling for value == null => skip observation and log it, or logging is done in getValue(..)
         LOG.debug("Value: {}", value.toString());
         // TODO implement using different templates in later version depending on the class of value

diff --git a/feeder/src/test/java/org/n52/sos/importer/feeder/ConfigurationTest.java b/feeder/src/test/java/org/n52/sos/importer/feeder/ConfigurationTest.java
@@ -33,6 +33,7 @@
 
 import org.apache.xmlbeans.XmlException;
 import org.hamcrest.CoreMatchers;
+import org.hamcrest.core.Is;
 import org.junit.Assert;
 import org.junit.Test;
 import org.x52North.sensorweb.sos.importer.x05.ColumnDocument.Column;
@@ -103,4 +104,20 @@ public void getColumnsForOmParameterShouldReturnCorrectValues() throws XmlExcept
         Assert.assertThat(column3.getMetadataArray(0).getValue(), CoreMatchers.is("BOOLEAN"));
     }
 
+    @Test
+    public void isNoDataValueDefinedAndMatchingShouldReturnFalseIfNotSet() throws XmlException, IOException {
+    	Configuration configuration = new Configuration(
+                "src/test/resources/feature_om-parameter/omparameter_set_as_related.xml");
+    	Column column = configuration.getColumnById(4);
+    	Assert.assertThat(configuration.isNoDataValueDefinedAndMatching(column, "14.8"), Is.is(false));
+    }
+
+    @Test
+    public void isNoDataValueDefinedAndMatchingShouldReturnTrueIfSetAndMatching() throws XmlException, IOException {
+    	Configuration configuration = new Configuration(
+                "src/test/resources/features/no_data_value_set.xml");
+    	Column column = configuration.getColumnById(4);
+    	Assert.assertThat(configuration.isNoDataValueDefinedAndMatching(column, "---"), Is.is(true));
+    }
+
 }
diff --git a/feeder/src/test/resources/features/no_data_value_set.xml b/feeder/src/test/resources/features/no_data_value_set.xml
@@ -0,0 +1,133 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<SosImportConfiguration xsi:schemaLocation="https://raw.githubusercontent.com/52North/sos-importer/master/bindings/src/main/resources/import-configuration.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://52north.org/sensorweb/sos/importer/0.5/">
+    <DataFile referenceIsARegularExpression="false">
+        <LocalFile>
+            <Path>src/test/resources/issue-057/data_obs.csv</Path>
+            <Encoding>UTF-8</Encoding>
+        </LocalFile>
+    </DataFile>
+    <SosMetadata>
+        <URL>http://localhost:8080/52n-sos-webapp/service</URL>
+        <Offering generate="true"/>
+        <Version>2.0.0</Version>
+        <Binding>POX</Binding>
+    </SosMetadata>
+    <CsvMetadata>
+        <ColumnAssignments>
+            <Column>
+                <Number>0</Number>
+                <Type>FOI</Type>
+            </Column>
+            <Column>
+                <Number>1</Number>
+                <Type>SENSOR</Type>
+            </Column>
+            <Column>
+                <Number>2</Number>
+                <Type>POSITION</Type>
+                <Metadata>
+                    <Key>TYPE</Key>
+                    <Value>COMBINATION</Value>
+                </Metadata>
+                <Metadata>
+                    <Key>PARSE_PATTERN</Key>
+                    <Value>LAT</Value>
+                </Metadata>
+                <Metadata>
+                    <Key>GROUP</Key>
+                    <Value>A</Value>
+                </Metadata>
+            </Column>
+            <Column>
+                <Number>3</Number>
+                <Type>POSITION</Type>
+                <Metadata>
+                    <Key>TYPE</Key>
+                    <Value>COMBINATION</Value>
+                </Metadata>
+                <Metadata>
+                    <Key>PARSE_PATTERN</Key>
+                    <Value>LON</Value>
+                </Metadata>
+                <Metadata>
+                    <Key>GROUP</Key>
+                    <Value>A</Value>
+                </Metadata>
+                <Metadata>
+                    <Key>POSITION_ALTITUDE</Key>
+                    <Value>0.0</Value>
+                </Metadata>
+                <Metadata>
+                    <Key>POSITION_EPSG_CODE</Key>
+                    <Value>4326</Value>
+                </Metadata>
+            </Column>
+            <Column>
+                <Number>4</Number>
+                <Type>MEASURED_VALUE</Type>
+                <Metadata>
+                    <Key>TYPE</Key>
+                    <Value>NUMERIC</Value>
+                </Metadata>
+                <Metadata>
+                    <Key>NO_DATA_VALUE</Key>
+                    <Value>---</Value>
+                </Metadata>
+            </Column>
+            <Column>
+                <Number>5</Number>
+                <Type>OBSERVED_PROPERTY</Type>
+            </Column>
+            <Column>
+                <Number>6</Number>
+                <Type>UOM</Type>
+            </Column>
+            <Column>
+                <Number>7</Number>
+                <Type>DATE_TIME</Type>
+                <Metadata>
+                    <Key>GROUP</Key>
+                    <Value>1</Value>
+                </Metadata>
+                <Metadata>
+                    <Key>PARSE_PATTERN</Key>
+                    <Value>M/dd/yyyy h.mm a</Value>
+                </Metadata>
+                <Metadata>
+                    <Key>TYPE</Key>
+                    <Value>COMBINATION</Value>
+                </Metadata>
+                <Metadata>
+                    <Key>TIME_ZONE</Key>
+                    <Value>1</Value>
+                </Metadata>
+            </Column>
+            <Column>
+                <Number>8</Number>
+                <Type>DO_NOT_EXPORT</Type>
+            </Column>
+            <Column>
+                <Number>9</Number>
+                <Type>DO_NOT_EXPORT</Type>
+            </Column>
+            <Column>
+                <Number>10</Number>
+                <Type>DO_NOT_EXPORT</Type>
+            </Column>
+        </ColumnAssignments>
+        <DecimalSeparator>.</DecimalSeparator>
+        <FirstLineWithData>1</FirstLineWithData>
+        <Parameter>
+            <CommentIndicator>#</CommentIndicator>
+            <ColumnSeparator>,</ColumnSeparator>
+            <TextIndicator>"</TextIndicator>
+        </Parameter>
+        <UseHeader>false</UseHeader>
+    </CsvMetadata>
+    <AdditionalMetadata>
+        <Metadata>
+            <Key>IMPORT_STRATEGY</Key>
+            <Value>SingleObservation</Value>
+        </Metadata>
+    </AdditionalMetadata>
+</SosImportConfiguration>