Transformer can now set minimum and maximum value of data

Values can be set with configuration file options 'minimum_value' and 'maximum_value'. Both options must present at the same time. Missing values in data are not changed.
fmidev · Jun 3, 2024 · 6606ec8 · 6606ec8
1 parent 9a5528a
commit 6606ec8
Show file tree

Hide file tree

Showing 4 changed files with 92 additions and 11 deletions.
diff --git a/doc/plugin-transformer.md b/doc/plugin-transformer.md
@@ -40,6 +40,14 @@ scale: define scale (default: 1)
 
     "scale" : "0.01"
 
+maximum_value: define the maximum value for the data. Missing values are not changed. Minimum and maximum need to be defined together.
+
+    "maximum_value" : "260"
+
+minimum_value: define the minimum value for the data. Missing values are not changed. Minimum and maximum need to be defined together.
+
+    "minimum_value" : "250"
+
 target_param: define target parameter name 
 
     "target_param" : "T-C"

diff --git a/himan-plugins/include/transformer.h b/himan-plugins/include/transformer.h
@@ -73,6 +73,8 @@ class transformer : public compiled_plugin, private compiled_plugin_base
 	std::unique_ptr<ensemble> itsEnsemble;
 	time_duration itsSourceForecastPeriod;
 	bool itsReadFromPreviousForecastIfNotFound;
+	double itsMinimumValue;
+	double itsMaximumValue;
 };
 
 // the class factory

diff --git a/himan-plugins/source/transformer.cpp b/himan-plugins/source/transformer.cpp
@@ -28,7 +28,7 @@ namespace transformergpu
 {
 template <typename T>
 void Process(shared_ptr<const himan::plugin_configuration> conf, shared_ptr<himan::info<T>> myTargetInfo,
-             shared_ptr<himan::info<T>> sourceInfo, double scale, double base);
+             shared_ptr<himan::info<T>> sourceInfo, double scale, double base, T min, T max);
 }
 #endif
 
@@ -50,7 +50,10 @@ transformer::transformer()
       itsParamDefinitionFromConfig(false),
       itsEnsemble(nullptr),
       itsSourceForecastPeriod(),
-      itsReadFromPreviousForecastIfNotFound(false)
+      itsReadFromPreviousForecastIfNotFound(false),
+      itsMinimumValue(himan::MissingDouble()),
+      itsMaximumValue(himan::MissingDouble())
+
 {
 	itsCudaEnabledCalculation = true;
 
@@ -353,6 +356,23 @@ void transformer::SetAdditionalParameters()
 		itsLogger.Trace("scale not specified, using default value 1.0");
 	}
 
+	if (itsConfiguration->Exists("minimum_value"))
+	{
+		itsMinimumValue = stod(itsConfiguration->GetValue("minimum_value"));
+	}
+
+	if (itsConfiguration->Exists("maximum_value"))
+	{
+		itsMaximumValue = stod(itsConfiguration->GetValue("maximum_value"));
+	}
+
+	if ((IsMissing(itsMinimumValue) && IsValid(itsMaximumValue)) ||
+	    (IsValid(itsMinimumValue) && IsMissing(itsMaximumValue)))
+	{
+		itsLogger.Fatal("Both minimum_value and maximum_value must be specified");
+		himan::Abort();
+	}
+
 	if (itsConfiguration->Exists("rotation"))
 	{
 		const auto spl = util::Split(itsConfiguration->GetValue("rotation"), ",");
@@ -839,21 +859,39 @@ void transformer::Calculate(shared_ptr<info<float>> myTargetInfo, unsigned short
 		{
 			deviceType = "GPU";
 
-			transformergpu::Process<float>(itsConfiguration, myTargetInfo, sourceInfo, itsScale, itsBase);
+			float min = IsValid(itsMinimumValue) ? static_cast<float>(itsMinimumValue) : MissingValue<float>();
+			float max = IsValid(itsMaximumValue) ? static_cast<float>(itsMaximumValue) : MissingValue<float>();
+
+			transformergpu::Process<float>(itsConfiguration, myTargetInfo, sourceInfo, itsScale, itsBase, min, max);
 		}
 		else
 #endif
 		{
 			transform(source.begin(), source.end(), result.begin(),
 			          [&](const float& value) { return fma(value, itsScale, itsBase); });
 
-			if (!IsMissing(itsChangeMissingTo))
+			if (IsValid(itsMinimumValue))
 			{
-				auto& vec = VEC(myTargetInfo);
-				replace_if(
-				    vec.begin(), vec.end(), [=](float d) { return IsMissing(d); }, itsChangeMissingTo);
+				float min = IsValid(itsMinimumValue) ? static_cast<float>(itsMinimumValue) : MissingValue<float>();
+				float max = IsValid(itsMaximumValue) ? static_cast<float>(itsMaximumValue) : MissingValue<float>();
+
+				for_each(result.begin(), result.end(),
+				         [&](float& value)
+				         {
+					         if (IsValid(value))
+					         {
+						         value = fmin(fmax(value, min), max);
+					         }
+				         });
 			}
 		}
+		if (!IsMissing(itsChangeMissingTo))
+		{
+			auto& vec = VEC(myTargetInfo);
+			replace_if(
+			    vec.begin(), vec.end(), [=](float d) { return IsMissing(d); }, itsChangeMissingTo);
+		}
+
 		myThreadedLogger.Info(fmt::format("[{}] Missing values: {}/{}", deviceType, myTargetInfo->Data().MissingCount(),
 		                                  myTargetInfo->Data().Size()));
 	}
@@ -954,7 +992,8 @@ void transformer::Calculate(shared_ptr<info<double>> myTargetInfo, unsigned shor
 	{
 		deviceType = "GPU";
 
-		transformergpu::Process(itsConfiguration, myTargetInfo, sourceInfo, itsScale, itsBase);
+		transformergpu::Process(itsConfiguration, myTargetInfo, sourceInfo, itsScale, itsBase, itsMinimumValue,
+		                        itsMaximumValue);
 	}
 	else
 #endif
@@ -966,6 +1005,18 @@ void transformer::Calculate(shared_ptr<info<double>> myTargetInfo, unsigned shor
 
 		transform(source.begin(), source.end(), result.begin(),
 		          [&](const double& value) { return fma(value, itsScale, itsBase); });
+
+		if (IsValid(itsMinimumValue))
+		{
+			for_each(result.begin(), result.end(),
+			         [&](double& value)
+			         {
+				         if (IsValid(value))
+				         {
+					         value = fmin(fmax(value, itsMinimumValue), itsMaximumValue);
+				         }
+			         });
+		}
 	}
 
 	if (!IsMissing(itsChangeMissingTo))

diff --git a/himan-plugins/source/transformer.cu b/himan-plugins/source/transformer.cu
@@ -14,11 +14,25 @@ __global__ void TransformerKernel(const T* __restrict__ d_source, T* __restrict_
 	}
 }
 
+template <typename T>
+__global__ void ClampKernel(T* __restrict__ d_dest, T min, T max, size_t N)
+{
+	const int idx = blockIdx.x * blockDim.x + threadIdx.x;
+
+	if (idx < N)
+	{
+		if (isfinite(d_dest[idx]))
+		{
+			d_dest[idx] = fmax(fmin(d_dest[idx], static_cast<T>(max)), static_cast<T>(min));
+		}
+	}
+}
+
 namespace transformergpu
 {
 template <typename T>
 void Process(std::shared_ptr<const himan::plugin_configuration> conf, std::shared_ptr<info<T>> myTargetInfo,
-             std::shared_ptr<info<T>> sourceInfo, double scale, double base)
+             std::shared_ptr<info<T>> sourceInfo, double scale, double base, T min, T max)
 {
 	cudaStream_t stream;
 
@@ -48,6 +62,12 @@ void Process(std::shared_ptr<const himan::plugin_configuration> conf, std::share
 	CUDA_CHECK(cudaStreamSynchronize(stream));
 
 	TransformerKernel<T><<<gridSize, blockSize, 0, stream>>>(d_source, d_dest, scale, base, N);
+
+	if (IsValid(min))
+	{
+		ClampKernel<T><<<gridSize, blockSize, 0, stream>>>(d_dest, min, max, N);
+	}
+
 	cuda::ReleaseInfo(myTargetInfo, d_dest, stream);
 
 	// block until the stream has completed
@@ -59,8 +79,8 @@ void Process(std::shared_ptr<const himan::plugin_configuration> conf, std::share
 	cudaStreamDestroy(stream);
 }
 template void Process(std::shared_ptr<const himan::plugin_configuration>, std::shared_ptr<info<double>>,
-                      std::shared_ptr<info<double>>, double, double);
+                      std::shared_ptr<info<double>>, double, double, double, double);
 template void Process(std::shared_ptr<const himan::plugin_configuration>, std::shared_ptr<info<float>>,
-                      std::shared_ptr<info<float>>, double, double);
+                      std::shared_ptr<info<float>>, double, double, float, float);
 
 }  // namespace transformergpu