Skip to content

Commit

Permalink
implement export all GridLAB files
Browse files Browse the repository at this point in the history
  • Loading branch information
@derrickoswald
derrickoswald committed Nov 1, 2017
1 parent f5abc6e commit dad4360
Show file tree
Hide file tree
Showing 7 changed files with 414 additions and 117 deletions.
4 changes: 2 additions & 2 deletions CIMWeb/src/main/scala/ch/ninecode/cim/cimweb/FileOperations.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,11 @@ class FileOperations extends RESTful
@Produces (Array (MediaType.APPLICATION_JSON))
def getFile (
@DefaultValue ("false") @MatrixParam ("debug") debug: String): Response =
getFile ("", "false", debug)
getFile ("/", "false", debug)

@GET
@Path ("{path:[^;]*}")
@Produces (Array (MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON, "application/zip"))
@Produces (Array (MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON, MediaType.APPLICATION_OCTET_STREAM, "text/plain", "text/csv", "application/zip"))
def getFile (
@PathParam ("path") path: String,
@DefaultValue ("false") @MatrixParam ("zip") zip: String,
Expand Down
284 changes: 255 additions & 29 deletions CIMWeb/src/main/scala/ch/ninecode/cim/cimweb/GridLAB.scala
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,28 @@
package ch.ninecode.cim.cimweb

import java.io.ByteArrayOutputStream
import java.io.StringReader
import java.nio.charset.StandardCharsets
import java.util.logging.Logger
import java.util.zip.ZipEntry
import java.util.zip.ZipOutputStream
import javax.ejb.Stateless
import javax.json.Json
import javax.json.JsonException
import javax.json.JsonObject
import javax.json.JsonStructure
import javax.resource.ResourceException
import javax.resource.cci.Connection
import javax.ws.rs.GET
import javax.ws.rs.POST
import javax.ws.rs.core.MediaType
import javax.ws.rs.Path
import javax.ws.rs.PathParam
import javax.ws.rs.Produces
import javax.resource.ResourceException
import javax.ws.rs.DefaultValue
import javax.ws.rs.MatrixParam
import javax.ws.rs.core.Response

import scala.collection.JavaConversions._
import scala.collection.JavaConverters._

import ch.ninecode.cim.connector.CIMFunction
import ch.ninecode.cim.connector.CIMInteractionSpec
Expand All @@ -28,46 +36,264 @@ class GridLAB extends RESTful
import GridLAB._

// get a righteous string for a glm filename
def glm_name (simulation: String): String =
def base_name (simulation: String): String =
{
val index = simulation.lastIndexOf ("/")
val suffix = if (simulation.endsWith (".json")) simulation.length - 4 else simulation.length
(if (-1 == index) simulation.substring (0, suffix) else simulation.substring (index + 1, suffix)) + ".glm"
val suffix = if (simulation.endsWith (".json")) simulation.length - 5 else simulation.length
if (-1 == index) simulation.substring (0, suffix) else simulation.substring (index + 1, suffix)
}

// get a righteous string for a glm filename
def glm_name (simulation: String): String =
{
base_name (simulation) + ".glm"
}

// get a righteous string for a zip filename
def zip_name (simulation: String): String =
{
base_name (simulation) + ".zip"
}

def create (connection: Connection, simulation: String): String =
{
val spec: CIMInteractionSpec = new CIMInteractionSpecImpl
spec.setFunctionName (CIMInteractionSpec.EXECUTE_CIM_FUNCTION)
val input = getInputRecord ("input record containing the function to run")
// set up the function with parameters
val gridlab = GridLABExportFunction (simulation)
input.asInstanceOf[map].put (CIMFunction.FUNCTION, gridlab)
val interaction = connection.createInteraction
val output = interaction.execute (spec, input)
if (null == output)
throw new ResourceException ("null is not a MappedRecord")
else
if (!output.getClass.isAssignableFrom (classOf [CIMMappedRecord]))
throw new ResourceException ("object of class %s is not a MappedRecord".format (output.getClass.toGenericString))
else
{
val record = output.asInstanceOf [CIMMappedRecord]
record.get (CIMFunction.RESULT).asInstanceOf [String]
}
}

def get (connection: Connection, file: String): String =
{
val spec: CIMInteractionSpec = new CIMInteractionSpecImpl
spec.setFunctionName (CIMInteractionSpec.EXECUTE_CIM_FUNCTION)
val input = getInputRecord ("input record containing the function to run")
val function = GetFileFunction (file)
input.asInstanceOf[map].put (CIMFunction.FUNCTION, function)
val interaction = connection.createInteraction
val output = interaction.execute (spec, input)
val record = output.asInstanceOf [CIMMappedRecord]
val xml = record.get (CIMFunction.RESULT).asInstanceOf [String]
interaction.close ()
if (xml.startsWith ("File does not exist:"))
null
else
xml
}

case class GatherOptions (
sim: Boolean, // include simulation
cim: Boolean, // include CIM file
in: Boolean, // include input files
out: Boolean, // include output files
all: Boolean // include everything
)
{
def gather: Boolean = sim || cim || in || out || all
}

def gather (connection: Connection, simulation: String, options: GatherOptions): Array[Byte] =
{
// get the simulation json file
val json = get (connection, simulation)
if (null != json)
// parse it
try
Json.createReader (new StringReader (json)).readObject match
{
case details: JsonObject ⇒

// get the simulation root directory
val sim = if (-1 == simulation.lastIndexOf ("/")) simulation else simulation.substring (simulation.lastIndexOf ("/") + 1)
val sim_root = simulation.substring (0, simulation.length - sim.length)

// get the gridlab root directory
val glm_file = details.getString ("glm")
if (null == glm_file)
return Array ()
val last = glm_file.lastIndexOf ("/")
val (root, name) = if (-1 == last)
(simulation.substring (0, simulation.length - sim.length), glm_file) // assumed relative to simulation file
else
{
val name = glm_file.substring (last + 1)
val second_last = glm_file.lastIndexOf ("/", last - 1)
if (-1 == second_last)
("", name)
else
(glm_file.substring (second_last + 1, glm_file.length - name.length), name)
}

// build the zip file
val bos = new ByteArrayOutputStream ()
val zos = new ZipOutputStream (bos)
zos.setLevel (9)

// store the glm
val glm = get (connection, sim_root + root + name)
if (null == glm)
return Array ()
else
{
zos.putNextEntry (new ZipEntry (root + name))
val data = glm.getBytes (StandardCharsets.UTF_8)
zos.write (data, 0, data.length)
}

// store the simulation
if (options.sim || options.all)
{
zos.putNextEntry (new ZipEntry (root + sim))
val data = json.getBytes (StandardCharsets.UTF_8)
zos.write (data, 0, data.length)
}

// store the CIM file
val cim = details.getString ("cim")
if ((null != cim ) && (options.cim || options.all))
{
val guts = get (connection, cim)
if (null != guts)
{
val name = if (-1 == cim.lastIndexOf ("/")) cim else cim.substring (cim.lastIndexOf ("/") + 1)
zos.putNextEntry (new ZipEntry (root + name))
val data = guts.getBytes (StandardCharsets.UTF_8)
zos.write (data, 0, data.length)
}
}

var dirs: Array[String] = Array()

// store the players
val players = details.getJsonArray ("players")
if (null != players)
{
for (element: JsonObject ← players.getValuesAs (classOf[JsonObject]).asScala) // ToDo: more robust checking
{
val player_file = element.getString ("file", "")
val player_name = if (-1 == player_file.lastIndexOf ("/")) player_file else player_file.substring (player_file.lastIndexOf ("/") + 1)
if (options.in || options.all)
{
val player = get (connection, sim_root + root + player_file)
if (null != player)
{
zos.putNextEntry (new ZipEntry (root + player_file))
val data = player.getBytes (StandardCharsets.UTF_8)
zos.write (data, 0, data.length)
}
}
else
{
val dir = player_file.substring (0, player_file.length - player_name.length)
if (dir != "" && !dirs.contains (dir))
{
zos.putNextEntry (new ZipEntry (root + dir))
dirs = dirs :+ dir
}
}
}
}

// store the recorders
val recorders = details.getJsonArray ("recorders")
if (null != recorders)
{
for (element: JsonObject ← recorders.getValuesAs (classOf[JsonObject]).asScala) // ToDo: more robust checking
{
val recorder_file = element.getString ("file", "")
val recorder_name = if (-1 == recorder_file.lastIndexOf ("/")) recorder_file else recorder_file.substring (recorder_file.lastIndexOf ("/") + 1)
if (options.out || options.all)
{
val recorder = get (connection, sim_root + root + recorder_file)
if (null != recorder)
{
zos.putNextEntry (new ZipEntry (root + recorder_file))
val data = recorder.getBytes (StandardCharsets.UTF_8)
zos.write (data, 0, data.length)
}
}
else
{
val dir = recorder_file.substring (0, recorder_file.length - recorder_name.length)
if (dir != "" && !dirs.contains (dir))
{
zos.putNextEntry (new ZipEntry (root + dir))
dirs = dirs :+ dir
}
}
}
}

// seal the zip file
zos.finish ()
zos.close ()
bos.toByteArray
case _ ⇒ Array()
}
catch
{
case je: JsonException ⇒ Array()
}
else
Array()
}

@GET
@Path ("{simulation:[^;]*}")
@Produces (Array (MediaType.APPLICATION_JSON))
@Produces (Array (MediaType.APPLICATION_JSON, "application/zip"))
def export (
@PathParam ("simulation") simulation: String // the name of the JSON simulation file on HDFS
): Response =
@PathParam ("simulation") simulation: String, // the name of the JSON simulation file on HDFS
@DefaultValue ("false") @MatrixParam ("sim") sim: String,
@DefaultValue ("false") @MatrixParam ("cim") cim: String,
@DefaultValue ("false") @MatrixParam ("in") in: String,
@DefaultValue ("false") @MatrixParam ("out") out: String,
@DefaultValue ("false") @MatrixParam ("all") all: String
): Response =
{
_Logger.info ("gridlab %s".format (simulation))
_Logger.info ("gridlab %s;all=%s".format (simulation, all))
val ret = new RESTfulJSONResult
val connection = getConnection (ret)
val response: Response = if (null != connection)
try
{
val spec: CIMInteractionSpec = new CIMInteractionSpecImpl
spec.setFunctionName (CIMInteractionSpec.EXECUTE_CIM_FUNCTION)
val input = getInputRecord ("input record containing the function to run")
// set up the function with parameters
val gridlab = GridLABExportFunction (if (simulation.startsWith ("/")) simulation else "/" + simulation)
input.asInstanceOf[map].put (CIMFunction.FUNCTION, gridlab)
val interaction = connection.createInteraction
val output = interaction.execute (spec, input)
if (null == output)
throw new ResourceException ("null is not a MappedRecord")
else
if (!output.getClass.isAssignableFrom (classOf [CIMMappedRecord]))
throw new ResourceException ("object of class %s is not a MappedRecord".format (output.getClass.toGenericString))
else
{
val record = output.asInstanceOf [CIMMappedRecord]
Response.ok (record.get (CIMFunction.RESULT).asInstanceOf [String], MediaType.APPLICATION_OCTET_STREAM)
.header ("content-disposition", "attachment; filename=%s".format (glm_name (simulation)))
val sim = if (simulation.startsWith ("/")) simulation else "/" + simulation
val options = GatherOptions (
sim = try { sim.toBoolean } catch { case _: Throwable => false },
cim = try { cim.toBoolean } catch { case _: Throwable => false },
in = try { in.toBoolean } catch { case _: Throwable => false },
out = try { out.toBoolean } catch { case _: Throwable => false },
all = try { all.toBoolean } catch { case _: Throwable => false })
if (options.gather)
{
val zip = gather (connection, sim, options)
if (zip.length > 0)
Response.ok (zip, "application/zip")
.header ("content-disposition", "attachment; filename=%s".format (zip_name (simulation)))
.build
}
else
Response.serverError ().entity ("zip construction failed").build
}
else
{
val glm = create (connection, sim)
Response.ok (glm, MediaType.APPLICATION_OCTET_STREAM)
.header ("content-disposition", "attachment; filename=%s".format (glm_name (simulation)))
.build
}
}
catch
{
Expand Down
2 changes: 1 addition & 1 deletion CIMWeb/src/main/scala/ch/ninecode/cim/cimweb/GridLABExportFunction.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ case class GridLABExportFunction (simulation: String) extends CIMWebFunction
else
{
val buffer = for (element: JsonObject ← array.getValuesAs (classOf[JsonObject]).asScala) // ToDo: more robust checking
yield Player (element.getString ("name", ""), element.getString ("parent", ""), element.getString ("player", ""))
yield Player (element.getString ("name", ""), element.getString ("parent", ""), element.getString ("file", ""))
buffer.toArray
}
}
Expand Down
11 changes: 10 additions & 1 deletion CIMWeb/src/main/scala/ch/ninecode/cim/cimweb/GridLABSimulateFunction.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,15 @@ import javax.json.JsonException
import javax.json.JsonObject
import javax.json.JsonStructure

import scala.collection.JavaConverters._

import org.apache.hadoop.fs.Path
import org.apache.hadoop.io.Text
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SparkSession

import ch.ninecode.cim.cimweb.RESTfulJSONResult.OK
import ch.ninecode.cim.cimweb.RESTfulJSONResult.FAIL

import ch.ninecode.cim.connector.CIMFunction.Return

/**
Expand Down Expand Up @@ -120,6 +121,14 @@ case class GridLABSimulateFunction (simulation: String) extends CIMWebFunction
{
val workdir_path = glm.substring (0, index1 + 1)
val file = glm.substring (index1 + 1, index2)
// erase all the recorder files
val recorders = details.getJsonArray ("recorders")
val root = glm.substring (0, index2 + 1)
for (element: JsonObject ← recorders.getValuesAs (classOf[JsonObject]).asScala) // ToDo: more robust checking
{
val recorder_file = element.getString ("file", "")
hdfs.delete (new Path (root, recorder_file), false)
}
val message = solve (workdir_path, spark.sparkContext.parallelize (Array (file)))
if (message == "")
new RESTfulJSONResult (OK, "gridlab simulation ran").getJSON
Expand Down
Loading

0 comments on commit dad4360

Please sign in to comment.