Merge pull request #180 from ie3-institute/to/#178-custom-node-uuids

To/#178 custom node uuids

Author: t-ober

.gitignore

@@ -146,6 +146,36 @@ dmypy.json
 # profiling data
 .prof
 
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+
 # End of https://www.toptal.com/developers/gitignore/api/python
 
 # Ignore Gradle project-specific cache directory

build.gradle

@@ -92,6 +92,8 @@ dependencies {
 	// Linter Scala //
 	implementation "com.sksamuel.scapegoat:scalac-scapegoat-plugin_${scalaBinaryVersion}:1.4.11" // scala scapegoat
 	scalaCompilerPlugin "com.sksamuel.scapegoat:scalac-scapegoat-plugin_${scalaBinaryVersion}:1.4.11" // scala scapegoat
+
+	implementation 'tech.units:indriya:2.1.3-SNAPSHOT'
 }
 
 /* scapegoat hook configuration

src/main/python/powerFactory2json/pf2jsonUtils.py

@@ -24,6 +24,7 @@
     ],
     'conElms': [],
     'nodes': [
+        "loc_name",
         "vtarget",
         "uknom",
         "GPSlat",

src/main/scala/edu/ie3/powerFactory2psdm/config/ConversionConfig.scala

@@ -7,6 +7,7 @@
 package edu.ie3.powerFactory2psdm.config
 
 import edu.ie3.powerFactory2psdm.config.ConversionConfig.{
+  NodeUuidMappingInformation,
   OutputConfig,
   StatGenModelConfigs
 }
@@ -19,6 +20,7 @@ import edu.ie3.powerFactory2psdm.config.model.{
 final case class ConversionConfig(
     gridName: String,
     modelConfigs: StatGenModelConfigs,
+    nodeMapping: Option[NodeUuidMappingInformation],
     output: OutputConfig
 )
 
@@ -42,6 +44,11 @@ object ConversionConfig {
       cosPhiSource: ParameterSource
   )
 
+  final case class NodeUuidMappingInformation(
+      filePath: String,
+      csvSeparator: String
+  )
+
   final case class OutputConfig(
       targetFolder: String,
       csvConfig: CsvConfig

src/main/scala/edu/ie3/powerFactory2psdm/converter/ConversionHelper.scala

@@ -80,4 +80,21 @@ object ConversionHelper {
         )
     }
 
+  /** Get all duplicates of the sequence.
+    *
+    * @param items
+    *   items to check for duplicates
+    * @tparam T
+    *   type of the items
+    * @return
+    *   sequence of duplicates
+    */
+  def getDuplicates[T](items: Seq[T]): Seq[T] = {
+    val uniqueItems = items.distinct
+    if (uniqueItems.size == items.size) {
+      return Seq.empty[T]
+    }
+    items.diff(uniqueItems)
+  }
+
 }

src/main/scala/edu/ie3/powerFactory2psdm/converter/GridConverter.scala

@@ -27,13 +27,17 @@ import edu.ie3.datamodel.models.input.system.{
   StorageInput
 }
 import edu.ie3.powerFactory2psdm.config.ConversionConfig
-import edu.ie3.powerFactory2psdm.config.ConversionConfig.StatGenModelConfigs
+import edu.ie3.powerFactory2psdm.config.ConversionConfig.{
+  NodeUuidMappingInformation,
+  StatGenModelConfigs
+}
+import edu.ie3.powerFactory2psdm.converter.NodeConverter.getNodeNameMapping
 import edu.ie3.powerFactory2psdm.converter.types.{
   LineTypeConverter,
   Transformer2WTypeConverter
 }
 import edu.ie3.powerFactory2psdm.model.{PreprocessedPfGridModel, RawPfGridModel}
-
+import java.util.UUID
 import scala.jdk.CollectionConverters.SetHasAsJava
 
 /** Functionalities to transform an exported and then parsed PowerFactory grid
@@ -50,7 +54,8 @@ case object GridConverter {
       config.modelConfigs.sRatedSource,
       config.modelConfigs.cosPhiSource
     )
-    val (gridElements, convertedNodes) = convertGridElements(grid)
+    val (gridElements, convertedNodes) =
+      convertGridElements(grid, config.nodeMapping)
     val participants =
       convertParticipants(grid, convertedNodes, config.modelConfigs)
     new JointGridContainer(
@@ -70,13 +75,16 @@ case object GridConverter {
     *   the raw parsed PowerFactoryGrid
     */
   def convertGridElements(
-      grid: PreprocessedPfGridModel
+      grid: PreprocessedPfGridModel,
+      nodeUuidMapping: Option[NodeUuidMappingInformation]
   ): (RawGridElements, Map[String, NodeInput]) = {
     val graph =
       GridGraphBuilder.build(grid.nodes, grid.lines ++ grid.switches)
     val nodeId2node = grid.nodes.map(node => (node.id, node)).toMap
     val subnets = SubnetBuilder.buildSubnets(graph, nodeId2node)
-    val nodes = NodeConverter.convertNodesOfSubnets(subnets)
+    val nodeId2Uuid =
+      nodeUuidMapping.map(getNodeNameMapping).getOrElse(Map.empty[String, UUID])
+    val nodes = NodeConverter.convertNodesOfSubnets(subnets, nodeId2Uuid)
     val lineTypes = grid.lineTypes
       .map(lineType => (lineType.id, LineTypeConverter.convert(lineType)))
       .toMap
@@ -112,7 +120,6 @@ case object GridConverter {
       ),
       nodes
     )
-
   }
 
   /** Convert system participants of the power factory grid.

src/main/scala/edu/ie3/powerFactory2psdm/converter/NodeConverter.scala

@@ -9,19 +9,17 @@ package edu.ie3.powerFactory2psdm.converter
 import edu.ie3.datamodel.models.OperationTime
 import edu.ie3.datamodel.models.input.{NodeInput, OperatorInput}
 import edu.ie3.datamodel.models.voltagelevels.VoltageLevel
-import edu.ie3.powerFactory2psdm.exception.pf.{
-  ConversionException,
-  GridConfigurationException
-}
-import edu.ie3.powerFactory2psdm.model.entity.{Node, Subnet}
-import edu.ie3.util.quantities.PowerSystemUnits.PU
+import edu.ie3.powerFactory2psdm.config.ConversionConfig.NodeUuidMappingInformation
+import edu.ie3.powerFactory2psdm.exception.pf.ConversionException
+import edu.ie3.powerFactory2psdm.model.entity.Subnet
 import edu.ie3.powerFactory2psdm.exception.pf.GridConfigurationException
 import edu.ie3.powerFactory2psdm.model.entity.Node
 import edu.ie3.powerFactory2psdm.util.QuantityUtils.RichQuantityDouble
 import org.locationtech.jts.geom.Point
-import tech.units.indriya.quantity.Quantities
 
+import java.io.IOException
 import java.util.UUID
+import scala.io.Source
 import scala.util.{Failure, Success, Try}
 
 object NodeConverter {
@@ -30,26 +28,44 @@ object NodeConverter {
     *
     * @param subnets
     *   subnets of the grid
+    * @param unsafeNodeId2Uuid
+    *   mapping form human readable identifier set in pf to the generated UUID
     * @return
     *   Map of node id to PSDM [[NodeInput]]
     */
-  def convertNodesOfSubnets(subnets: List[Subnet]): Map[String, NodeInput] = {
-    subnets.flatMap(subnet => convertNodesOfSubnet(subnet)).toMap
+  def convertNodesOfSubnets(
+      subnets: List[Subnet],
+      unsafeNodeId2Uuid: Map[String, UUID]
+  ): Map[String, NodeInput] = {
+    subnets
+      .flatMap(subnet => convertNodesOfSubnet(subnet, unsafeNodeId2Uuid))
+      .toMap
   }
 
   /** Converts all nodes within a subnet to PSDM [[NodeInput]] s
     *
     * @param subnet
     *   the subnet with reference to all PF nodes that live within
+    * @param unsafeNodeId2Uuid
+    *   mapping form human readable identifier set in pf to the generated UUID
     * @return
     *   list of all converted [[NodeInput]]
     */
   def convertNodesOfSubnet(
-      subnet: Subnet
+      subnet: Subnet,
+      unsafeNodeId2Uuid: Map[String, UUID]
   ): Set[(String, NodeInput)] =
     subnet.nodes
       .map(node =>
-        (node.id, NodeConverter.convertNode(node, subnet.id, subnet.voltLvl))
+        (
+          node.id,
+          NodeConverter.convertNode(
+            node,
+            subnet.id,
+            subnet.voltLvl,
+            unsafeNodeId2Uuid
+          )
+        )
       )
 
   /** Converts a PowerFactory node into a PSDM node.
@@ -60,18 +76,22 @@ object NodeConverter {
     *   subnet id the node is assigned to
     * @param voltLvl
     *   voltage level of the node
+    * @param unsafeNodeId2Uuid
+    *   mapping form human readable identifier set in pf to the generated UUID
     * @return
     *   a PSDM [[NodeInput]]
     */
   def convertNode(
       node: Node,
       subnetId: Int,
-      voltLvl: VoltageLevel
+      voltLvl: VoltageLevel,
+      unsafeNodeId2Uuid: Map[String, UUID]
   ): NodeInput = {
     val geoPosition: Point = CoordinateConverter.convert(node.lat, node.lon)
     val slack = isSlack(node)
+    val uuid = unsafeNodeId2Uuid.getOrElse(node.unsafeId, UUID.randomUUID())
     new NodeInput(
-      UUID.randomUUID(),
+      uuid,
       node.id,
       OperatorInput.NO_OPERATOR_ASSIGNED,
       OperationTime.notLimited(),
@@ -83,6 +103,69 @@ object NodeConverter {
     )
   }
 
+  /** Creates a Map that maps from unsafe node ids to uuids from a csv file that
+    * is used for node conversion. This can be used to keep uuids of the nodes
+    * consistent between original data and the converted grid.
+    *
+    * @param nodeUuidMappingInformation
+    *   necessary mapping information
+    * @return
+    *   Map from unsafe node id to uuid
+    */
+  def getNodeNameMapping(
+      nodeUuidMappingInformation: NodeUuidMappingInformation
+  ): Map[String, UUID] = {
+    // parse csv file
+    val bufferedSource = Source.fromFile(nodeUuidMappingInformation.filePath)
+    val lines = bufferedSource.getLines()
+
+    lines.take(1).next.split(nodeUuidMappingInformation.csvSeparator) match {
+      case Array("uuid", "id") =>
+      case _ =>
+        throw new IOException(
+          "Invalid CSV header. We expect the header of the node name mapping file to be \"uuid\" [CSV Sep.] \"id\"."
+        )
+    }
+
+    val idsAndUuids = lines.map { line =>
+      line
+        .split(nodeUuidMappingInformation.csvSeparator)
+        .map(_.trim) match {
+        case Array(uuidString, id) =>
+          val uuid = Try(UUID.fromString(uuidString)) match {
+            case Failure(exception) =>
+              throw new IllegalArgumentException(
+                s"UUID: $uuidString on line: $line is not a valid UUID.",
+                exception
+              )
+            case Success(uuid) => uuid
+          }
+          (id, uuid)
+        case Array(_) =>
+          throw new IOException(
+            s"Invalid format on csv line INDEX_HERE. Every line should have exactly two elements."
+          )
+      }
+    }.toVector
+
+    // check for duplicates in ids or uuids
+    val (ids, uuids) = idsAndUuids.unzip
+    val duplicateIds = ConversionHelper.getDuplicates(ids)
+    if (duplicateIds.nonEmpty) {
+      throw ConversionException(
+        f"There are the following duplicate ids in the node id to uuid mapping: $duplicateIds"
+      )
+    }
+    val duplicateUuids = ConversionHelper.getDuplicates(uuids)
+    if (duplicateIds.nonEmpty) {
+      throw ConversionException(
+        f"There are the following duplicate uuids in the node id to uuid mapping: $duplicateUuids"
+      )
+    }
+    bufferedSource.close
+    idsAndUuids.toMap
+  }
+
   /** Checks if a node is a slack node by checking if there is an external grid
     * connected to the node.
     *

src/main/scala/edu/ie3/powerFactory2psdm/converter/types/LineTypeConverter.scala

@@ -62,7 +62,7 @@ object LineTypeConverter extends LazyLogging {
 
     // sanity check of total line length versus aggregated line length of all corresponding line sections
     lineLength - aggregatedLineSectionLength match {
-      case x if abs(x) < 1e-9 =>
+      case _ if abs(aggregatedLineSectionLength / lineLength - 1) < 1e-6 =>
       case x if x < 0 =>
         logger.error(
           s"The line length of line: $lineId is smaller than the aggregated length of line sections by ${(1 - (lineLength / aggregatedLineSectionLength)) * 100}% which distorts results. This should be prevented by PF and therefore not happen."
@@ -94,20 +94,20 @@ object LineTypeConverter extends LazyLogging {
     )
 
     weightedLineTypes.foldLeft(emptyLineType)((averageType, current) => {
-      val currentLine = current._2
+      val currentLineType = current._2
       val weightingFactor = current._1 / lineLength
       new LineTypeInput(
         averageType.getUuid,
         averageType.getId,
-        averageType.getB.add(currentLine.getB.multiply(weightingFactor)),
-        averageType.getG.add(currentLine.getG.multiply(weightingFactor)),
-        averageType.getR.add(currentLine.getR.multiply(weightingFactor)),
-        averageType.getX.add(currentLine.getX.multiply(weightingFactor)),
-        if (averageType.getiMax().isLessThan(currentLine.getiMax()))
+        averageType.getB.add(currentLineType.getB.multiply(weightingFactor)),
+        averageType.getG.add(currentLineType.getG.multiply(weightingFactor)),
+        averageType.getR.add(currentLineType.getR.multiply(weightingFactor)),
+        averageType.getX.add(currentLineType.getX.multiply(weightingFactor)),
+        if (averageType.getiMax().isLessThan(currentLineType.getiMax()))
           averageType.getiMax()
-        else currentLine.getiMax(),
+        else currentLineType.getiMax(),
         if (averageType.getvRated().equals(Double.MaxValue.asKiloVolt))
-          currentLine.getvRated()
+          currentLineType.getvRated()
         else averageType.getvRated()
       )
     })