Skip to content

Commit

Permalink
Fix non-numberic values. (#198)
Browse files Browse the repository at this point in the history
  • Loading branch information
@pradh
pradh authored Mar 3, 2023
1 parent 7874392 commit 62c0ab6
Show file tree
Hide file tree
Showing 4 changed files with 19 additions and 9 deletions.
2 changes: 1 addition & 1 deletion tool/src/test/resources/org/datacommons/tool/genmcf/localidresolution/output/report.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,4 +59,4 @@
"observationAbout": false,
"allowNanSvobs": false
}
}
}
6 changes: 5 additions & 1 deletion ...c/test/resources/org/datacommons/tool/genmcf/measurementresult/output/summary_report.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -435,7 +435,11 @@ <h2>
<td>
<div></div>
</td>
<td style="max-width:none;text-align: -webkit-center;"><b>Charts for non-numeric types are not supported yet</b></td>
<td style="max-width:none;text-align: -webkit-center;"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:jfreesvg="http://www.jfree.org/jfreesvg/svg" width="500" height="250" text-rendering="auto" shape-rendering="auto">
<defs><clipPath id="testclip-0"><path d="M 0 0 L 500 0 L 500 250 L 0 250 L 0 0 Z "/></clipPath>
<clipPath id="testclip-1"><path d="M 52 10 L 52 227 L 488 227 L 488 10 Z "/></clipPath>
</defs>
<rect x="0" y="0" width="500" height="250" style="fill: rgb(255,255,255); fill-opacity: 1.0" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/><rect x="52" y="10" width="436" height="217" style="fill: rgb(255,255,255); fill-opacity: 1.0" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/><line x1="52" y1="231" x2="488" y2="231" style="stroke-width: 0.5;stroke: rgb(128,128,128);stroke-opacity: 1.0;stroke-linecap: square;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/><g transform="matrix(1,0,0,1,0,0)"><text x="23" y="245.64" style="fill: rgb(64,64,64); fill-opacity: 1.0; font-family: sans-serif; font-size: 10px;" clip-path="url(#testclip-0)">2020-01-04</text></g><line x1="52" y1="233" x2="52" y2="231" style="stroke-width: 0.5;stroke: rgb(128,128,128);stroke-opacity: 1.0;stroke-linecap: square;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/><g transform="matrix(1,0,0,1,0,0)"><text x="241" y="245.64" style="fill: rgb(64,64,64); fill-opacity: 1.0; font-family: sans-serif; font-size: 10px;" clip-path="url(#testclip-0)">2020-01-05</text></g><line x1="270" y1="233" x2="270" y2="231" style="stroke-width: 0.5;stroke: rgb(128,128,128);stroke-opacity: 1.0;stroke-linecap: square;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/><line x1="48" y1="10" x2="48" y2="227" style="stroke-width: 0.5;stroke: rgb(128,128,128);stroke-opacity: 1.0;stroke-linecap: square;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/><g transform="matrix(1,0,0,1,0,0)"><text x="20" y="231.14" style="fill: rgb(64,64,64); fill-opacity: 1.0; font-family: sans-serif; font-size: 10px;" clip-path="url(#testclip-0)">41.0</text></g><line x1="46" y1="227" x2="48" y2="227" style="stroke-width: 0.5;stroke: rgb(128,128,128);stroke-opacity: 1.0;stroke-linecap: square;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/><g transform="matrix(1,0,0,1,0,0)"><text x="20" y="209.44" style="fill: rgb(64,64,64); fill-opacity: 1.0; font-family: sans-serif; font-size: 10px;" clip-path="url(#testclip-0)">41.2</text></g><line x1="46" y1="205.3" x2="48" y2="205.3" style="stroke-width: 0.5;stroke: rgb(128,128,128);stroke-opacity: 1.0;stroke-linecap: square;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/><g transform="matrix(1,0,0,1,0,0)"><text x="20" y="187.74" style="fill: rgb(64,64,64); fill-opacity: 1.0; font-family: sans-serif; font-size: 10px;" clip-path="url(#testclip-0)">41.4</text></g><line x1="46" y1="183.6" x2="48" y2="183.6" style="stroke-width: 0.5;stroke: rgb(128,128,128);stroke-opacity: 1.0;stroke-linecap: square;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/><g transform="matrix(1,0,0,1,0,0)"><text x="20" y="166.04" style="fill: rgb(64,64,64); fill-opacity: 1.0; font-family: sans-serif; font-size: 10px;" clip-path="url(#testclip-0)">41.6</text></g><line x1="46" y1="161.9" x2="48" y2="161.9" style="stroke-width: 0.5;stroke: rgb(128,128,128);stroke-opacity: 1.0;stroke-linecap: square;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/><g transform="matrix(1,0,0,1,0,0)"><text x="20" y="144.34" style="fill: rgb(64,64,64); fill-opacity: 1.0; font-family: sans-serif; font-size: 10px;" clip-path="url(#testclip-0)">41.8</text></g><line x1="46" y1="140.2" x2="48" y2="140.2" style="stroke-width: 0.5;stroke: rgb(128,128,128);stroke-opacity: 1.0;stroke-linecap: square;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/><g transform="matrix(1,0,0,1,0,0)"><text x="20" y="122.64" style="fill: rgb(64,64,64); fill-opacity: 1.0; font-family: sans-serif; font-size: 10px;" clip-path="url(#testclip-0)">42.0</text></g><line x1="46" y1="118.5" x2="48" y2="118.5" style="stroke-width: 0.5;stroke: rgb(128,128,128);stroke-opacity: 1.0;stroke-linecap: square;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/><g transform="matrix(1,0,0,1,0,0)"><text x="20" y="100.94" style="fill: rgb(64,64,64); fill-opacity: 1.0; font-family: sans-serif; font-size: 10px;" clip-path="url(#testclip-0)">42.2</text></g><line x1="46" y1="96.8" x2="48" y2="96.8" style="stroke-width: 0.5;stroke: rgb(128,128,128);stroke-opacity: 1.0;stroke-linecap: square;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/><g transform="matrix(1,0,0,1,0,0)"><text x="20" y="79.24" style="fill: rgb(64,64,64); fill-opacity: 1.0; font-family: sans-serif; font-size: 10px;" clip-path="url(#testclip-0)">42.4</text></g><line x1="46" y1="75.1" x2="48" y2="75.1" style="stroke-width: 0.5;stroke: rgb(128,128,128);stroke-opacity: 1.0;stroke-linecap: square;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/><g transform="matrix(1,0,0,1,0,0)"><text x="20" y="57.54" style="fill: rgb(64,64,64); fill-opacity: 1.0; font-family: sans-serif; font-size: 10px;" clip-path="url(#testclip-0)">42.6</text></g><line x1="46" y1="53.4" x2="48" y2="53.4" style="stroke-width: 0.5;stroke: rgb(128,128,128);stroke-opacity: 1.0;stroke-linecap: square;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/><g transform="matrix(1,0,0,1,0,0)"><text x="20" y="35.84" style="fill: rgb(64,64,64); fill-opacity: 1.0; font-family: sans-serif; font-size: 10px;" clip-path="url(#testclip-0)">42.8</text></g><line x1="46" y1="31.7" x2="48" y2="31.7" style="stroke-width: 0.5;stroke: rgb(128,128,128);stroke-opacity: 1.0;stroke-linecap: square;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/><g transform="matrix(1,0,0,1,0,0)"><text x="20" y="14.14" style="fill: rgb(64,64,64); fill-opacity: 1.0; font-family: sans-serif; font-size: 10px;" clip-path="url(#testclip-0)">43.0</text></g><line x1="46" y1="10" x2="48" y2="10" style="stroke-width: 0.5;stroke: rgb(128,128,128);stroke-opacity: 1.0;stroke-linecap: square;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/><line x1="52" y1="10" x2="52" y2="227" style="stroke-width: 0.5;stroke: rgb(255,255,255);stroke-opacity: 1.0;stroke-linejoin: bevel;stroke-dasharray: 2.0, 2.0;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-1)"/><line x1="270" y1="10" x2="270" y2="227" style="stroke-width: 0.5;stroke: rgb(255,255,255);stroke-opacity: 1.0;stroke-linejoin: bevel;stroke-dasharray: 2.0, 2.0;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-1)"/><line x1="52" y1="227" x2="488" y2="227" style="stroke-width: 0.5;stroke: rgb(255,255,255);stroke-opacity: 1.0;stroke-linejoin: bevel;stroke-dasharray: 2.0, 2.0;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-1)"/><line x1="52" y1="205.3" x2="488" y2="205.3" style="stroke-width: 0.5;stroke: rgb(255,255,255);stroke-opacity: 1.0;stroke-linejoin: bevel;stroke-dasharray: 2.0, 2.0;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-1)"/><line x1="52" y1="183.6" x2="488" y2="183.6" style="stroke-width: 0.5;stroke: rgb(255,255,255);stroke-opacity: 1.0;stroke-linejoin: bevel;stroke-dasharray: 2.0, 2.0;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-1)"/><line x1="52" y1="161.9" x2="488" y2="161.9" style="stroke-width: 0.5;stroke: rgb(255,255,255);stroke-opacity: 1.0;stroke-linejoin: bevel;stroke-dasharray: 2.0, 2.0;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-1)"/><line x1="52" y1="140.2" x2="488" y2="140.2" style="stroke-width: 0.5;stroke: rgb(255,255,255);stroke-opacity: 1.0;stroke-linejoin: bevel;stroke-dasharray: 2.0, 2.0;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-1)"/><line x1="52" y1="118.5" x2="488" y2="118.5" style="stroke-width: 0.5;stroke: rgb(255,255,255);stroke-opacity: 1.0;stroke-linejoin: bevel;stroke-dasharray: 2.0, 2.0;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-1)"/><line x1="52" y1="96.8" x2="488" y2="96.8" style="stroke-width: 0.5;stroke: rgb(255,255,255);stroke-opacity: 1.0;stroke-linejoin: bevel;stroke-dasharray: 2.0, 2.0;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-1)"/><line x1="52" y1="75.1" x2="488" y2="75.1" style="stroke-width: 0.5;stroke: rgb(255,255,255);stroke-opacity: 1.0;stroke-linejoin: bevel;stroke-dasharray: 2.0, 2.0;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-1)"/><line x1="52" y1="53.4" x2="488" y2="53.4" style="stroke-width: 0.5;stroke: rgb(255,255,255);stroke-opacity: 1.0;stroke-linejoin: bevel;stroke-dasharray: 2.0, 2.0;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-1)"/><line x1="52" y1="31.7" x2="488" y2="31.7" style="stroke-width: 0.5;stroke: rgb(255,255,255);stroke-opacity: 1.0;stroke-linejoin: bevel;stroke-dasharray: 2.0, 2.0;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-1)"/><line x1="52" y1="10" x2="488" y2="10" style="stroke-width: 0.5;stroke: rgb(255,255,255);stroke-opacity: 1.0;stroke-linejoin: bevel;stroke-dasharray: 2.0, 2.0;shape-rendering:crispEdges;" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-1)"/><g style="fill: rgb(255,0,0); fill-opacity: 1.0; stroke: none" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-1)"><path d="M 272 118.5 C 272 119.6 271.1 120.5 270 120.5 C 268.9 120.5 268 119.6 268 118.5 C 268 117.4 268.9 116.5 270 116.5 C 271.1 116.5 272 117.4 272 118.5 Z "/></g><g style="stroke-width: 1.0;stroke: rgb(255,0,0);stroke-opacity: 1.0;stroke-linecap: square;shape-rendering:geometricPrecision;; fill: none" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-1)"><path d="M 272 118.5 C 272 119.6 271.1 120.5 270 120.5 C 268.9 120.5 268 119.6 268 118.5 C 268 117.4 268.9 116.5 270 116.5 C 271.1 116.5 272 117.4 272 118.5 Z "/></g><rect x="52" y="10" width="436" height="217" style="stroke-width: 0.5;stroke: rgb(0,0,0);stroke-opacity: 1.0;stroke-linecap: round;stroke-linejoin: round;shape-rendering:crispEdges;; fill: none" transform="matrix(1,0,0,1,0,0)" clip-path="url(#testclip-0)"/></svg></td>
</tr>
</tbody>
</table>
Expand Down
14 changes: 8 additions & 6 deletions util/src/main/java/org/datacommons/util/PlaceSeriesSummary.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,6 @@ public ValueType getValueType() {
}

public String getTimeSeriesSVGChart() {

if (getValueType() != ValueType.NUMBER) {
return "<b>Charts for non-numeric types are not supported yet</b>";
}

TimeSeries timeSeries = new TimeSeries("ts");

// this.timeSeries is kept sorted with a TreeMap, so we simply add the
Expand All @@ -83,14 +78,21 @@ public String getTimeSeriesSVGChart() {

LocalDateTime localDateTime = StringUtil.getValidISO8601Date(timeSeriesDataPoint.getKey());
if (localDateTime == null) continue;

DataPoint dp = timeSeriesDataPoint.getValue();
if (SeriesSummary.getTypeOfDataPoint(dp) != ValueType.NUMBER) continue;

timeSeries.addOrUpdate(
new Day(
localDateTime.getDayOfMonth(),
localDateTime.getMonthValue(),
localDateTime.getYear()),
getValueOfDataPointAsNumber(timeSeriesDataPoint.getValue()));
getValueOfDataPointAsNumber(dp));
}

if (timeSeries.getItemCount() == 0) {
return "<b>Charts for non-numeric types are not supported yet</b>";
}
return StatVarSummary.constructSVGChartFromTimeSeries(timeSeries);
}

Expand Down
6 changes: 5 additions & 1 deletion util/src/main/java/org/datacommons/util/StatChecker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -423,6 +423,7 @@ protected static void checkSigmaDivergence(
// Only add data points to the counter of the greatest standard deviation that it belongs to.
// ie. if the data point is beyond 3 std deviation, only add it to that counter.
for (DataPoint dp : timeSeries) {
if (SeriesSummary.getTypeOfDataPoint(dp) != ValueType.NUMBER) continue;
double val = SeriesSummary.getValueOfDataPointAsNumber(dp);
if (Math.abs(val - meanAndStdDev.mean) > 3 * meanAndStdDev.stdDev) {
sigma3Counter.addProblemPoints(dp);
Expand All @@ -446,6 +447,7 @@ private static MeanAndStdDev getStats(List<DataPoint> timeSeries) {
double sum = 0;
double sumSqDev = 0;
for (DataPoint dp : timeSeries) {
if (SeriesSummary.getTypeOfDataPoint(dp) != ValueType.NUMBER) continue;
double val = SeriesSummary.getValueOfDataPointAsNumber(dp);
if (weights > 0) {
sumSqDev += 1 * weights / 1 / (weights + 1) * Math.pow((1 / weights * sum - val), 2);
Expand All @@ -472,8 +474,10 @@ protected static void checkPercentFluctuations(
// Don't try to compare between times because this is a Sawtooth
if (dp.getValuesCount() > 1) return;
if (dp.getValuesCount() == 0) continue;
if (SeriesSummary.getTypeOfDataPoint(dp) != ValueType.NUMBER) continue;
double currVal = SeriesSummary.getValueOfDataPointAsNumber(dp);
if (baseDataPoint != null) {
if (baseDataPoint != null
&& SeriesSummary.getTypeOfDataPoint(baseDataPoint) == ValueType.NUMBER) {
double currDelta;
double baseVal = SeriesSummary.getValueOfDataPointAsNumber(baseDataPoint);
if (baseVal == 0) {
Expand Down

0 comments on commit 62c0ab6

Please sign in to comment.