diff --git a/README.md b/README.md index 36334e6..178a756 100644 --- a/README.md +++ b/README.md @@ -181,3 +181,5 @@ You will encounter these terms in the code and code comments: * **Visual Model**: Representation of the form used to fill in a notice, found in the `notice-types` folder of SDK * **Conceptual Model**: An intermediate representation made of fields and nodes, based on the SDK `fields.json` * **Physical Model**: The representation of a notice in XML, see "XML Generation" +* **UI**: User Interface, in the editor demo this means the forms, buttons, links in the browser +* **metadata**: In the editor demo this is generally SDK data \ No newline at end of file diff --git a/src/main/java/eu/europa/ted/eforms/noticeeditor/EformsNoticeEditorApp.java b/src/main/java/eu/europa/ted/eforms/noticeeditor/EformsNoticeEditorApp.java index 1c6f364..8b485fa 100644 --- a/src/main/java/eu/europa/ted/eforms/noticeeditor/EformsNoticeEditorApp.java +++ b/src/main/java/eu/europa/ted/eforms/noticeeditor/EformsNoticeEditorApp.java @@ -32,12 +32,15 @@ public class EformsNoticeEditorApp implements CommandLineRunner { @Value("${eforms.sdk.versions}") private List supportedSdks; + @Value("${eforms.sdk.autodownload}") + private boolean autoDownload; + public static void main(final String[] args) { logger.info("STARTING eForms Notice Editor Demo Application"); // See README.md on how to run server. // https://spring.io/guides/gs/serving-web-content/ - // Here you have access to command line args. + // Here you have access to command line arguments. // logger.debug("args={}", Arrays.toString(args)); SpringApplication.run(EformsNoticeEditorApp.class, args); @@ -48,6 +51,19 @@ public void run(String... args) throws Exception { Validate.notEmpty(eformsSdkDir, "Undefined eForms SDK path"); Validate.notNull(supportedSdks, "Undefined supported SDK versions"); + logger.info("SDK autoDownload: {}", autoDownload); + if (autoDownload) { + // This automatically downloads the supported officially SDKs. + // You can test any SDK (even release candidates) by doing a git clone and putting the SDK + // files manually inside the folder where it would normally be downloaded. + autoDownloadSupportedSdks(); + } else { + logger.info( + "Not automatically downloading SDKs, put files manually or set autoDownload to true in application.yaml"); + } + } + + private void autoDownloadSupportedSdks() { for (final String sdkVersion : supportedSdks) { try { SdkDownloader.downloadSdk(new SdkVersion(sdkVersion), Path.of(eformsSdkDir)); diff --git a/src/main/java/eu/europa/ted/eforms/noticeeditor/controller/XmlRestController.java b/src/main/java/eu/europa/ted/eforms/noticeeditor/controller/XmlRestController.java index 8ab1a54..6f20739 100644 --- a/src/main/java/eu/europa/ted/eforms/noticeeditor/controller/XmlRestController.java +++ b/src/main/java/eu/europa/ted/eforms/noticeeditor/controller/XmlRestController.java @@ -3,6 +3,7 @@ import java.util.Optional; import javax.servlet.http.HttpServletResponse; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.AsyncConfigurer; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; @@ -21,17 +22,32 @@ public class XmlRestController implements AsyncConfigurer { @Autowired private XmlWriteService xmlService; + /** + * Enriches the XML for human readability but it becomes invalid. Also adds .dot files and other + * debug files in target. + */ + @Value("${xml.generation.debug}") + private boolean debug; + + @Value("${xml.generation.skipIfEmpty}") + private boolean skipIfNoValue; + + @Value("${xml.generation.sortXmlElements}") + private boolean sortXmlElements; + /** * Save: Takes notice as JSON and builds notice XML. The SDK version is in the notice metadata. */ @RequestMapping(value = "/notice/save/validation/none", method = RequestMethod.POST, produces = SdkService.MIME_TYPE_XML, consumes = SdkService.MIME_TYPE_JSON) - public void saveNotice(final HttpServletResponse response, final @RequestBody String noticeJson) + public void saveNotice(final HttpServletResponse response, + final @RequestBody String noticeJson) throws Exception { - // Enriches the XML for human readability but it becomes invalid. - // Also adds .dot files in target. - final boolean debug = false; - xmlService.saveNoticeAsXml(Optional.of(response), noticeJson, debug); + // For the XML generation config booleans, see application.yaml + xmlService.saveNoticeAsXml(Optional.of(response), noticeJson, + debug, + skipIfNoValue, + sortXmlElements); } /** @@ -42,7 +58,6 @@ public void saveNotice(final HttpServletResponse response, final @RequestBody St produces = SdkService.MIME_TYPE_XML, consumes = SdkService.MIME_TYPE_JSON) public void saveNoticeAndXsdValidate(final HttpServletResponse response, final @RequestBody String noticeJson) throws Exception { - final boolean debug = false; xmlService.validateUsingXsd(Optional.of(response), noticeJson, debug); } @@ -55,7 +70,6 @@ public void saveNoticeAndXsdValidate(final HttpServletResponse response, produces = SdkService.MIME_TYPE_XML, consumes = SdkService.MIME_TYPE_JSON) public void saveNoticeAndCvsValidate(final HttpServletResponse response, final @RequestBody String noticeJson) throws Exception { - final boolean debug = false; xmlService.validateUsingCvs(Optional.of(response), noticeJson, debug); } } diff --git a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/ConceptTreeField.java b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/ConceptTreeField.java index 567eeae..e93e559 100644 --- a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/ConceptTreeField.java +++ b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/ConceptTreeField.java @@ -1,9 +1,13 @@ package eu.europa.ted.eforms.noticeeditor.helper.notice; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + /** - * Conceptual field. Leaf in the conceptual tree. This holds non-metadata field information and the - * field id. This is not an SDK field, this only points to an SDK field to reference metadata. + * Conceptual field. Leaf in the conceptual tree. This holds non-metadata field information like the + * value and the associated SDK field id. This is not an SDK field, this only points to an SDK field + * to reference the metadata. A field item cannot have child items! */ +@JsonPropertyOrder({"idUnique", "counter", "fieldId", "value"}) public class ConceptTreeField extends ConceptTreeItem { private final String value; @@ -14,7 +18,9 @@ public ConceptTreeField(final String idUnique, final String idInSdkFieldsJson, f } /** - * For convenience and to make it clear that the ID in the SDK is the field ID in this case. + * For convenience and to make it clear that the ID in the SDK is the field ID in this case. It + * can be used to get general information about the field (data from fields.json). This does not + * include the counter. */ public String getFieldId() { return idInSdkFieldsJson; diff --git a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/ConceptTreeItem.java b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/ConceptTreeItem.java index b67a230..5df3f0b 100644 --- a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/ConceptTreeItem.java +++ b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/ConceptTreeItem.java @@ -2,19 +2,20 @@ import java.util.Objects; import org.apache.commons.lang3.Validate; +import com.fasterxml.jackson.annotation.JsonIgnore; /** - * Abstract item holding common information. References SDK metadata. + * Abstract item holding common conceptual information for a notice. References SDK metadata. */ public abstract class ConceptTreeItem { /** - * Unique identifier among children at same level. + * Unique identifier among children at same level. Counter excluded. */ private final String idUnique; /** * This id is not unique as some concept items can be repeatead while still have the same metadata - * (pointing to same field or same node multiple times). + * (pointing to same field or same node multiple times). A counter helps to differentiate them. */ protected final String idInSdkFieldsJson; @@ -32,12 +33,17 @@ protected ConceptTreeItem(final String idUnique, final String idInSdkFieldsJson, } /** - * @return Unique identifier among children at same level. + * @return Unique identifier among children at same level. Counter excluded. */ public String getIdUnique() { return idUnique; } + @JsonIgnore // This will be covered by getFieldId and getNodeId + public String getIdInSdkFieldsJson() { + return idInSdkFieldsJson; + } + public int getCounter() { return counter; } @@ -50,6 +56,7 @@ public String toString() { @Override public int hashCode() { + // Important: the counter is taken into account, this matters for repeatable items. return Objects.hash(counter, idInSdkFieldsJson, idUnique); } @@ -65,6 +72,7 @@ public boolean equals(Object obj) { return false; } ConceptTreeItem other = (ConceptTreeItem) obj; + // Important: the counter is taken into account, this matters for repeatable items. return counter == other.counter && Objects.equals(idInSdkFieldsJson, other.idInSdkFieldsJson) && Objects.equals(idUnique, other.idUnique); } diff --git a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/ConceptTreeNode.java b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/ConceptTreeNode.java index 04682ed..4c60907 100644 --- a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/ConceptTreeNode.java +++ b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/ConceptTreeNode.java @@ -1,15 +1,22 @@ package eu.europa.ted.eforms.noticeeditor.helper.notice; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; import org.apache.commons.lang3.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; /** * Conceptual node. This holds non-metadata information about a node. This is not an SDK node, this - * only points to an SDK node to reference metadata. + * only points to an SDK node to reference metadata. A node can have child items! */ +@JsonPropertyOrder({"idUnique", "counter", "nodeId", "repeatable", "conceptFields", "conceptNodes"}) public class ConceptTreeNode extends ConceptTreeItem { + private static final Logger logger = LoggerFactory.getLogger(ConceptTreeNode.class); + private final List conceptFields = new ArrayList<>(); /** @@ -32,18 +39,22 @@ public ConceptTreeNode(final String idUnique, final String idInSdkFieldsJson, fi /** * @param item The item to add. + * @param sb A string builder passed for debugging purposes */ @edu.umd.cs.findbugs.annotations.SuppressFBWarnings(value = "ITC_INHERITANCE_TYPE_CHECKING", justification = "Spotbugs is confused, the check is done on the passed item, not the class.") - public final void addConceptItem(final ConceptTreeItem item) { + public final Optional addConceptItem(final ConceptTreeItem item, + final StringBuilder sb) { if (item instanceof ConceptTreeNode) { - addConceptNode((ConceptTreeNode) item, true); - } else if (item instanceof ConceptTreeField) { - addConceptField((ConceptTreeField) item); - } else { - throw new RuntimeException( - String.format("Unexpected item type for concept item=%s", item.getIdUnique())); + final boolean strict = true; + return addConceptNode((ConceptTreeNode) item, strict, sb, "addConceptItem"); } + if (item instanceof ConceptTreeField) { + addConceptField((ConceptTreeField) item, sb); + return Optional.of(this); // Always added on self. + } + throw new RuntimeException( + String.format("Unexpected item type for concept item=%s", item.getIdUnique())); } public Optional findFirstByConceptNodeId(final String nodeId) { @@ -71,52 +82,195 @@ public String getNodeId() { return idInSdkFieldsJson; } - public final void addConceptField(final ConceptTreeField conceptField) { + public final void addConceptField(final ConceptTreeField conceptField, + final StringBuilder sb) { Validate.notNull(conceptField); conceptFields.add(conceptField); + logger.debug("Added concept field uniqueId={} to fieldId={}", conceptField.getFieldId(), + this.getIdUnique(), + conceptField.getIdUnique()); + sb.append("Added concept field: ") + .append(conceptField.getIdUnique()) + .append(" to ").append(this.getIdUnique()) + .append('\n'); } - public final void addConceptNode(final ConceptTreeNode conceptNode, final boolean strict) { - Validate.notNull(conceptNode); - final String otherNodeId = conceptNode.getNodeId(); - final String thisNodeId = getNodeId(); - if (otherNodeId.equals(thisNodeId)) { + /** + * @param cn The concept node to add + * @param strictAdd When true, if the item is already contained it will fail + * @param sb A string builder passed for debugging purposes + * + * @return The conceptual node to which the passed element was added, this must be used on the + * outside of the call + */ + public final Optional addConceptNode(final ConceptTreeNode cn, + final boolean strictAdd, + final StringBuilder sb, final String originalOfCall) { + Validate.notNull(cn); + + final String nodeIdToAdd = cn.getNodeId(); + final String nodeIdSelf = getNodeId(); + if (nodeIdToAdd.equals(nodeIdSelf)) { // Detect cycle, we have a tree, we do not want a graph, cannot self reference! throw new RuntimeException( String.format("Cannot have child=%s that is same as parent=%s (cycle), self reference.", - otherNodeId, thisNodeId)); + nodeIdToAdd, nodeIdSelf)); } - if (conceptNode.isRepeatable()) { + + if (cn.isRepeatable()) { // It is repeatable, meaning it can exist multiple times, just add it. - conceptNodes.add(conceptNode); - return; + addConceptNodePrivate(cn, "repeatable"); + sb.append("Added concept node (repeatable): ").append(cn.getIdUnique()).append(" to ") + .append(this.getIdUnique()).append('\n'); + logger.debug("Added concept node uniqueId={} to nodeId={}", cn.getIdUnique(), + this.getNodeId()); + return Optional.of(cn); } - // It is not repeatable. Is it already contained? - final boolean contained = conceptNodes.contains(conceptNode); + // It is not repeatable. + // Is it already contained? + final boolean cnAlreadyContained = containsNodeOrNodeId(cn); + + if (strictAdd) { + // Strict add. + if (cnAlreadyContained) { + if (containsNodeEquals(cn)) { + // It should not already be contained as an exact copy, this is a problem! + throw new RuntimeException(String.format( + "Conceptual model: node is not repeatable, it would be added twice, " + + "id=%s (nodeId=%s), parentId=%s, originOfCall=%s", + cn.getIdUnique(), cn.getNodeId(), this.getIdUnique(), originalOfCall)); + } else { + // It is already contained but is not the exact same object, there is another concept node + // with the same node id, so it that cannot be repeated. + // In this case we have to fuse the concept nodes. + // Fuse with existing node. + final Optional existingCnOpt = findFirstByConceptNodeId(nodeIdSelf); + if (existingCnOpt.isPresent()) { + final ConceptTreeNode cnExisting = existingCnOpt.get(); + // Fuse fields. + for (final ConceptTreeField cnField : cn.getConceptFields()) { + cnExisting.addConceptField(cnField, sb); + } + // Fuse nodes. + for (final ConceptTreeNode cnNode : cn.getConceptNodes()) { + cnExisting.addConceptNode(cnNode, strictAdd, sb, "fusion2"); + } + return Optional.of(cnExisting); + } + throw new RuntimeException(String.format( + "Conceptual model: node not found by nodeId=%s, parentId=%s, originOfCall=%s", + cn.getNodeId(), this.getIdUnique(), originalOfCall)); + } + } + // Not repeatable and not already contained, add it. + addConceptNodePrivate(cn, "not repeatable (strictAdd)"); + sb.append("Added concept node (not repeatable): ").append(cn.getIdUnique()).append(" to ") + .append(this.getIdUnique()).append('\n'); + logger.debug("Added concept node uniqueId={} to nodeId={}", cn.getIdUnique(), + this.getNodeId()); + return Optional.of(cn); + } + + // Non-strict add. + // We can add even if it is already contained. + // if (!cnAlreadyContained) { + // Add if not contained. Do not complain if already contained. + + // We DO NOT want to add the entire thing blindly. + + // Example: add X + + // conceptNodes -> empty, X is not there just add X (no problem) + + // conceptNodes -> (X -> Y -> 4141) + + // Example: add X but X is already there + // conceptNodes -> (X -> Y -> Z -> 4242) - // It should not already be contained. - if (strict) { - if (contained) { - throw new RuntimeException(String.format( - "Conceptual model: node is not repeatable " - + "but it is added twice, id=%s (nodeId=%s), parentId=%s", - conceptNode.getIdUnique(), conceptNode.getNodeId(), this.getIdUnique())); + // We want to fuse / fusion of branches: + // conceptNodes -> X -> Y -> 4141 + // .......................-> Z -> 4242 + + // So X and Y are reused (fused), and Z is attached to Y + + if (cnAlreadyContained) { + final int indexOfCn = conceptNodes.indexOf(cn); + if (indexOfCn >= 0) { + // Iterative fusion logic: + // X could be already contained, but some child item like Y may not be there yet. + final ConceptTreeNode existingCn = conceptNodes.get(indexOfCn); + // We know that the branches are uni-dimensional (flat), so a simple for loop with return + // should work. At least for the known cases this works. + for (final ConceptTreeNode cn2 : cn.getConceptNodes()) { + final Optional cnToWhichItWasAdded = + existingCn.addConceptNode(cn2, strictAdd, sb, "fusion1"); + if (cnToWhichItWasAdded.isPresent()) { + logger.debug("Fusion for uniqueId={}", cn2.getIdUnique()); + sb.append("Fusion for uniqueId=").append(cn2.getIdUnique()).append('\n'); + return cnToWhichItWasAdded; + } + } + return Optional.empty(); + } + } else { + // Not contained yet, add it. + addConceptNodePrivate(cn, "not repeatable (bis)"); + sb.append("Added concept node (not repeatable)(bis): ").append(cn.getIdUnique()) + .append(" to ") + .append(this.getIdUnique()).append('\n'); + logger.debug("Added concept node uniqueId={} to nodeId={}", cn.getIdUnique(), + this.getNodeId()); + return Optional.of(cn); // It was add on this item. + } + return Optional.empty(); + } + + /** + * @return true if it contains the exact same concept node (at the level of the children, not + * recursive) + */ + private boolean containsNodeEquals(final ConceptTreeNode cn) { + return conceptNodes.contains(cn); + } + + /** + * @return true if it contains the exact same concept node or a concept node with the same node id + * (at the level of the children, not recursive) + */ + private boolean containsNodeOrNodeId(final ConceptTreeNode cn) { + return conceptNodes.contains(cn) || conceptNodes.stream() + .filter(item -> item.getNodeId().equals(cn.getNodeId())).count() > 0; + } + + /** + * Every add on this collection must go through this to ensure coherence. NOT RECURSIVE! + * + * @param cn The concept node to add + * @param originOfAdd The origin of the add call, put in the error messages or logs For internal + * usage only, used to centralise add check adds. + */ + private boolean addConceptNodePrivate(final ConceptTreeNode cn, final String originOfAdd) { + if (!cn.isRepeatable()) { + // This catches unexpected behaviour, probably an algorithms or bad data is to blame. + // The part of the code which lead here is added to the exception. + if (containsNodeOrNodeId(cn)) { + // Throw an exception as otherwise the conceptual model would be broken from here on. + throw new RuntimeException( + String.format( + "Attempting to add already contained non-repeatable node: %s, originOfAdd=%s", + cn.getNodeId(), originOfAdd)); } - conceptNodes.add(conceptNode); - } else if (!contained) { - // Non-strict. - // Add if not contained. Do not complain if already contained. - conceptNodes.add(conceptNode); } + return conceptNodes.add(cn); // The only place we use .add on this list in this class! } public List getConceptFields() { - return conceptFields; + return Collections.unmodifiableList(conceptFields); // Unmodifiable to avoid side-effects! } public List getConceptNodes() { - return conceptNodes; + return Collections.unmodifiableList(conceptNodes); // Unmodifiable to avoid side-effects! } public boolean isRepeatable() { @@ -125,7 +279,8 @@ public boolean isRepeatable() { @Override public String toString() { - return "ConceptTreeNode [conceptFields=" + conceptFields + ", conceptNodes=" + conceptNodes + return "ConceptTreeNode id=" + this.idInSdkFieldsJson + " [conceptFields=" + conceptFields + + ", conceptNodes=" + conceptNodes + ", repeatable=" + repeatable + "]"; } diff --git a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/ConceptualModel.java b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/ConceptualModel.java index 867b05e..3a8ac6e 100644 --- a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/ConceptualModel.java +++ b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/ConceptualModel.java @@ -6,28 +6,36 @@ import java.util.List; import java.util.Optional; import org.apache.commons.lang3.Validate; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import eu.europa.ted.eforms.noticeeditor.util.GraphvizDotTool; import eu.europa.ted.eforms.noticeeditor.util.JavaTools; +import eu.europa.ted.eforms.noticeeditor.util.JsonUtils; import eu.europa.ted.eforms.sdk.SdkVersion; /** - * The conceptual model (CM) is an intermediary model that is between the visual and the physical - * model. It holds a tree made of conceptual node and conceptual field instances. + *

The Conceptual Model (CM)

+ * + *

+ * An intermediary model that is between the visual and the physical model for the given notice + * data. It holds a tree made of conceptual node and conceptual field instances. + *

+ * *

* In this model the tree items must reference SDK nodes or fields so that SDK metadata can be * retrieved! *

+ * *

* There are no tree items which do not point to SDK field or node, so if a visual group is not - * pointing to a node all the children must be moved to the closes parent which points to a node in - * the conceptual hierarchy. In other words this is one step closer to the physical representation. + * pointing to a node all the children must be moved to the closest parent which points to a node in + * the conceptual hierarchy. In other words this is one step closer to the physical model. *

*/ public class ConceptualModel { - static final String ND_ROOT = "ND-Root"; - static final String ND_ROOT_EXTENSION = "ND-RootExtension"; + static final String ND_ROOT = FieldsAndNodes.ND_ROOT; + static final String ND_ROOT_EXTENSION = FieldsAndNodes.ND_ROOT_EXTENSION; /** * Notice field id having the eformsSdkVersion as a value. @@ -39,6 +47,11 @@ public class ConceptualModel { */ public static final String FIELD_ID_NOTICE_SUB_TYPE = "OPP-070-notice"; + /** + * Notice field id defining the notice sub type list attribute. + */ + public static final String FIELD_ID_NOTICE_SUB_TYPE_LIST = "OPP-070-notice-List"; + /** * Sector of activity. */ @@ -74,11 +87,14 @@ public ConceptTreeNode getTreeRootNode() { public final String getNoticeSubType() { // HARDCODED LOGIC. final List conceptNodes = treeRootNode.getConceptNodes(); + Validate.notEmpty(conceptNodes, "conceptNodes list is empty!"); + final Optional rootExtOpt = conceptNodes.stream() .filter(item -> ND_ROOT_EXTENSION.equals(item.getNodeId())).findFirst(); if (rootExtOpt.isEmpty()) { throw new RuntimeException(String.format("Conceptual model: Expecting to find root extension " - + "in conceptual model! Missing important nodeId=%s", ND_ROOT_EXTENSION)); + + "in conceptual model! Missing important nodeId=%s, conceptNodes=%s", ND_ROOT_EXTENSION, + conceptNodes)); } final ConceptTreeNode rootExtension = rootExtOpt.get(); @@ -94,7 +110,13 @@ public final String getNoticeSubType() { @Override public String toString() { - return "ConceptualModel [rootNode=" + treeRootNode + "]"; + try { + return JsonUtils.getStandardJacksonObjectMapper() + .writerWithDefaultPrettyPrinter() + .writeValueAsString(this.treeRootNode); + } catch (JsonProcessingException ex) { + throw new RuntimeException(ex); + } } /** @@ -127,7 +149,7 @@ public void writeDotFile(final FieldsAndNodes fieldsAndNodes) { // Visualizing it can help understand how it works or find problems. final boolean includeFields = true; final String dotText = this.toDot(fieldsAndNodes, includeFields); - final Path pathToFolder = Path.of("target/dot/"); + final Path pathToFolder = Path.of("target", "dot"); Files.createDirectories(pathToFolder); final Path pathToFile = pathToFolder.resolve(this.getNoticeSubType() + "-concept.dot"); JavaTools.writeTextFile(pathToFile, dotText); @@ -158,9 +180,7 @@ private static void toDotRec(final FieldsAndNodes fieldsAndNodes, final StringBu final String childId = childNode.getIdUnique() + "_" + childNode.getNodeId(); GraphvizDotTool.appendEdge("", color, - cnIdUnique, childId, // concept node -> concept node - sb); toDotRec(fieldsAndNodes, sb, childNode, includeFields); @@ -171,9 +191,7 @@ private static void toDotRec(final FieldsAndNodes fieldsAndNodes, final StringBu // This makes the tree a lot more bushy and can be hard to read. for (final ConceptTreeField cf : cn.getConceptFields()) { GraphvizDotTool.appendEdge(edgeLabel, GraphvizDotTool.COLOR_BLUE, - cnIdUnique, cf.getIdUnique() + "=" + cf.getValue(), // node -> field - sb); } } diff --git a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/FieldsAndNodes.java b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/FieldsAndNodes.java index dffa9c1..1d09729 100644 --- a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/FieldsAndNodes.java +++ b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/FieldsAndNodes.java @@ -17,8 +17,10 @@ import eu.europa.ted.eforms.sdk.SdkVersion; /** - * Holds JSON data of the SDK "fields.json" file. Reuse this after construction. As with all SDK - * data this is associated with an SDK version. + *

+ * Holds the entire JSON data of the SDK "fields.json" file. Reuse this after construction. As with + * all SDK data this is associated with an SDK version. + *

*/ public class FieldsAndNodes { @@ -26,15 +28,26 @@ public class FieldsAndNodes { private static final String ND_PREFIX = "ND-"; public static final String ND_ROOT = "ND-Root"; + public static final String ND_ROOT_EXTENSION = "ND-RootExtension"; public static final String FIELD_OR_NODE_ID_KEY = "id"; private static final String FIELDS_JSON_SDK_VERSION = "sdkVersion"; public static final String XPATH_RELATIVE = "xpathRelative"; public static final String XPATH_ABSOLUTE = "xpathAbsolute"; + public static final String ATTRIBUTES = "attributes"; + public static final String ATTRIBUTE_OF = "attributeOf"; + public static final String ATTRIBUTE_NAME = "attributeName"; + + public static final String ID = "id"; + public static final String FIELD_TYPE = "type"; + public static final String PRESET_VALUE = "presetValue"; + /** * Sort order. * - *

Since SDK 1.7, but data is only correct since SDK 1.8

+ *

+ * Since SDK 1.7, but data is only correct since SDK 1.8 + *

*/ public static final String XSD_SEQUENCE_ORDER_KEY = "xsdSequenceOrder"; @@ -46,8 +59,8 @@ public class FieldsAndNodes { public static final String FIELD_PARENT_NODE_ID = "parentNodeId"; public static final String NODE_PARENT_NODE_ID = "parentId"; - private static final String FIELD_REPEATABLE = "repeatable"; - private static final String NODE_REPEATABLE = "repeatable"; + public static final String FIELD_REPEATABLE = "repeatable"; + public static final String NODE_REPEATABLE = "repeatable"; private final Map fieldById; private final Map nodeById; @@ -163,6 +176,33 @@ public SdkVersion getSdkVersion() { return sdkVersion; } + /** + * Provided for convenience for the unit tests. + * + * @return The xpath absolute of the field, if the field does not exist it throws an exception + */ + public String getFieldType(final String fieldId) { + return getFieldType(this.getFieldById(fieldId)); + } + + /** + * Provided for convenience for the unit tests. + * + * @return The xpath absolute of the field, if the field does not exist it throws an exception + */ + public String getFieldXpathAbs(final String fieldId) { + return getFieldXpathAbs(this.getFieldById(fieldId)); + } + + /** + * Provided for convenience for the unit tests. + * + * @return The xpath relative of the field, if the field does not exist it throws an exception + */ + public String getFieldXpathRel(final String fieldId) { + return getFieldXpathRel(this.getFieldById(fieldId)); + } + private static SdkVersion parseSdkVersion(final JsonNode fieldsJsonRoot) { // Example: "sdkVersion" : "eforms-sdk-1.3.2", final String text = fieldsJsonRoot.get(FIELDS_JSON_SDK_VERSION).asText(null); @@ -214,6 +254,18 @@ public static boolean isNodeRepeatableStatic(final JsonNode nodeMeta) { return JsonUtils.getBoolStrict(nodeMeta, NODE_REPEATABLE); } + public static String getFieldXpathAbs(final JsonNode fieldMeta) { + return JsonUtils.getTextStrict(fieldMeta, XPATH_ABSOLUTE); + } + + public static String getFieldXpathRel(final JsonNode fieldMeta) { + return JsonUtils.getTextStrict(fieldMeta, XPATH_RELATIVE); + } + + public static String getFieldType(final JsonNode fieldMeta) { + return JsonUtils.getTextStrict(fieldMeta, FIELD_TYPE); + } + public static void setFieldFlatCodeList(final ObjectMapper mapper, final ObjectNode field, final String codelistId) { final ObjectNode codeList = mapper.createObjectNode(); diff --git a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/PhysicalModel.java b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/PhysicalModel.java index 4d187c1..0c308e0 100644 --- a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/PhysicalModel.java +++ b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/PhysicalModel.java @@ -2,14 +2,17 @@ import static eu.europa.ted.eforms.noticeeditor.util.JsonUtils.getTextStrict; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.ParserConfigurationException; @@ -28,6 +31,7 @@ import eu.europa.ted.eforms.noticeeditor.helper.VersionHelper; import eu.europa.ted.eforms.noticeeditor.sorting.NoticeXmlTagSorter; import eu.europa.ted.eforms.noticeeditor.util.EditorXmlUtils; +import eu.europa.ted.eforms.noticeeditor.util.JavaTools; import eu.europa.ted.eforms.noticeeditor.util.JsonUtils; import eu.europa.ted.eforms.noticeeditor.util.XmlUtils; import eu.europa.ted.eforms.noticeeditor.util.XpathUtils; @@ -35,9 +39,13 @@ import eu.europa.ted.eforms.sdk.SdkVersion; /** - * The physical model (PM) holds the XML representation. This class also provides static methods to - * build the physical model from the conceptual model and a method to serialize it to XML text. The - * SDK version is taken into account. + *

The Physical Model (PM)

+ * + *

+ * Holds the XML document representation. This class also provides static methods to build the + * physical model from the conceptual model and a method to serialize it to XML text. The SDK + * version is taken into account. + *

* *

* Note that the XML elements are sorted, reordered, according to information found in the SDK. @@ -51,25 +59,23 @@ public class PhysicalModel { private static final String CBC_ID = "cbc:ID"; // Notice id, related to BT-701-notice. private static final String XMLNS = "xmlns"; - /** - * A special case that we have to solve. HARDCODED. TODO - */ - static final String NATIONAL = "national"; - private static final String NODE_XPATH_RELATIVE = "xpathRelative"; public static final String FIELD_CODE_LIST = "codeList"; - private static final String FIELD_TYPE_CODE = "code"; private static final String FIELD_XPATH_RELATIVE = "xpathRelative"; - private static final String FIELD_TYPE = "type"; private static final String XML_ATTR_EDITOR_COUNTER_SELF = "editorCounterSelf"; private static final String XML_ATTR_EDITOR_FIELD_ID = "editorFieldId"; private static final String XML_ATTR_EDITOR_NODE_ID = "editorNodeId"; - private static final String XML_ATTR_SCHEME_NAME = "schemeName"; - private static final String XML_ATTR_LIST_NAME = "listName"; - private static final String XPATH_TEMP_REPLACEMENT = "~"; // ONE CHAR ONLY! + public static final String XML_ATTR_LIST_NAME = "listName"; + public static final String XML_ATTR_SCHEME_NAME = "schemeName"; + + /** + * This character should not be part of the reserved xpath technical characters. It should also no + * be part of any xpath that is used in the SDK. + */ + private static final char XPATH_TEMP_REPLACEMENT = '~'; // ONE CHAR ONLY! /** * W3C Document Object Model (DOM), holds the XML representation. This can be queried using xpath @@ -86,7 +92,8 @@ public class PhysicalModel { * @param xpathInst Used for xpath evaluation * @param fieldsAndNodes Holds SDK field and node metadata * @param mainXsdPathOpt Path to the main XSD file to use, may be empty if the feature is not - * supported in an older SDK + * supported in an older SDK. This can be used later for XSD validation of the physical + * model */ public PhysicalModel(final Document document, final XPath xpathInst, final FieldsAndNodes fieldsAndNodes, final Optional mainXsdPathOpt) { @@ -140,17 +147,19 @@ private Node getSdkVersionElement() { } /** - * This is provided for convenience for the unit tests. + * This is provided for convenience for the unit tests. This should only be used on elements (not + * attributes). * *

* Evaluates xpath and returns a nodelist. Note: this works when the required notice values are * present as some xpath may rely on their presence (codes, indicators, ...). *

* - * @param contextElem The XML context element in which the xpath is evaluated - * @param xpathExpr The XPath expression relative to the passed context + * @param xpathExpr The XPath expression used to find elements * @param idForError An identifier which is shown in case of errors - * @return The result of evaluating the XPath expression as a list of elements + * + * @return The result of evaluating the XPath expression as a list of elements (does not work for + * attributes) */ List evaluateXpathForTests(final String xpathExpr, String idForError) { return XmlUtils.evaluateXpathAsElemList(this.xpathInst, this.getDomDocument(), xpathExpr, @@ -158,7 +167,33 @@ List evaluateXpathForTests(final String xpathExpr, String idForError) { } /** - * @param indented True if the xml text should be indented, false otherwise. + * This is provided for convenience for the unit tests. This works on elements and attributes. + * + * @param xpathExpr The XPath expression used to find elements + * @param idForError An identifier which is shown in case of errors + * + * @return The result of evaluating the XPath expression as a NodeList (works for attributes) + */ + NodeList evaluateXpathForTestsAsNodeList(final String xpathExpr, String idForError) { + return XmlUtils.evaluateXpathAsNodeList(this.xpathInst, this.getDomDocument(), xpathExpr, + idForError); + } + + /** + * This is provided for convenience for the unit tests. This works on elements and attributes. + * + * @param xpathExpr The XPath expression used to find elements + * @param idForError An identifier which is shown in case of errors + * + * @return The result of evaluating the XPath expression as a List of Node + */ + List evaluateXpathForTestsAsListOfNode(final String xpathExpr, String idForError) { + return XmlUtils.evaluateXpathAsListOfNode(this.xpathInst, this.getDomDocument(), xpathExpr, + idForError); + } + + /** + * @param indented True if the XML text should be indented, false otherwise. * * @return The XML as text. */ @@ -172,7 +207,7 @@ public String toString() { } /** - * Builds the physical model. + * Builds the physical model from the conceptual model and SDK field and nodes information. * * @param conceptModel The conceptual model from the previous step * @param fieldsAndNodes Information about SDK fields and nodes @@ -182,6 +217,8 @@ public String toString() { * production * @param buildFields Allows to disable field building, for debugging purposes. Note that if xpath * relies on the presence of fields or attribute of fields this could be problematic + * @param sortXml Sorts the XML according to the SDK xsdSequenceOrder if true, else keeps the raw + * order * * @return The physical model as an object containing the XML with a few extras */ @@ -192,7 +229,8 @@ public static PhysicalModel buildPhysicalModel(final ConceptualModel conceptMode final FieldsAndNodes fieldsAndNodes, final Map noticeInfoBySubtype, final Map documentInfoByType, final boolean debug, final boolean buildFields, - final Path sdkRootFolder) + final Path sdkRootFolder, + final boolean sortXml) throws ParserConfigurationException, SAXException, IOException { logger.info("Attempting to build physical model."); @@ -204,51 +242,65 @@ public static PhysicalModel buildPhysicalModel(final ConceptualModel conceptMode logger.info("XML DOM validating={}", safeDocBuilder.isValidating()); final Document xmlDoc = safeDocBuilder.newDocument(); - xmlDoc.setXmlStandalone(true); final DocumentTypeInfo docTypeInfo = getDocumentTypeInfo(noticeInfoBySubtype, documentInfoByType, conceptModel); final String rootElementType = docTypeInfo.getRootElementTagName(); // Create the root element, top level element. - final Element xmlDocRoot = createElemXml(xmlDoc, rootElementType); + final StringBuilder sb = new StringBuilder(512); + final Element xmlDocRoot = createElemXml(xmlDoc, rootElementType, debug, "", sb, "node"); xmlDoc.appendChild(xmlDocRoot); // TEDEFO-1426 // For the moment do as if it was there. final XPath xpathInst = setXmlNamespaces(docTypeInfo, xmlDocRoot); - // Attempt to put schemeName first. - // buildPhysicalModelXmlRec(fieldsAndNodes, doc, concept.getRoot(), rootElem, debug, - // buildFields, - // 0, true, xPathInst); - if (debug) { + // Write dot file about conceptual model. conceptModel.writeDotFile(fieldsAndNodes); + JavaTools.writeTextFile(Path.of("target", "debug", "conceptual-model.json"), + conceptModel.toString()); } // Recursion: start with the concept root. final ConceptTreeNode conceptualModelTreeRootNode = conceptModel.getTreeRootNode(); - final boolean onlyIfPriority = false; final int depth = 0; buildPhysicalModelRec(xmlDoc, fieldsAndNodes, conceptualModelTreeRootNode, xmlDocRoot, debug, - buildFields, depth, onlyIfPriority, xpathInst); + buildFields, depth, sb, xpathInst); + logger.info("Done building unsorted physical model."); - // Reorder the physical model. + // Reorder / sort the physical model. // The location of the XSDs is given in the SDK and could vary by SDK version. final SdkVersion sdkVersion = fieldsAndNodes.getSdkVersion(); final Path pathToSpecificSdk = sdkRootFolder.resolve(sdkVersion.toStringWithoutPatch()); final NoticeXmlTagSorter sorter = new NoticeXmlTagSorter(xpathInst, docTypeInfo, pathToSpecificSdk, fieldsAndNodes); - sorter.sortXml(xmlDocRoot); + try { + if (sortXml) { + logger.info("Attempting to sort physical model."); + sorter.sortXml(xmlDocRoot); + } - final Optional mainXsdPathOpt = sorter.getMainXsdPathOpt(); - if (mainXsdPathOpt.isPresent()) { - Validate.isTrue(mainXsdPathOpt.get().toFile().exists(), "File does not exist: mainXsdPath=%s", - mainXsdPathOpt); + final Optional mainXsdPathOpt = sorter.getMainXsdPathOpt(); + if (mainXsdPathOpt.isPresent()) { + Validate.isTrue(mainXsdPathOpt.get().toFile().exists(), + "File does not exist: mainXsdPath=%s", + mainXsdPathOpt); + } + if (debug) { + final Path path = Path.of("target", "debug"); + Files.createDirectories(path); + JavaTools.writeTextFile(path.resolve("physical-model.txt"), sb.toString()); + } + + return new PhysicalModel(xmlDoc, xpathInst, fieldsAndNodes, mainXsdPathOpt); + } catch (final Exception e) { + final String xmlAsText = EditorXmlUtils.asText(xmlDoc, true); + logger.error("Problem sorting XML: {}", xmlAsText); // Log the entire XML. + throw e; } - return new PhysicalModel(xmlDoc, xpathInst, fieldsAndNodes, mainXsdPathOpt); } /** @@ -264,8 +316,8 @@ public static PhysicalModel buildPhysicalModel(final ConceptualModel conceptMode */ private static void buildPhysicalModelRec(final Document doc, final FieldsAndNodes fieldsAndNodes, final ConceptTreeNode conceptElem, final Element xmlNodeElem, final boolean debug, - final boolean buildFields, final int depth, final boolean onlyIfPriority, - final XPath xpathInst) { + final boolean buildFields, final int depth, + final StringBuilder sb, final XPath xpathInst) { Validate.notNull(conceptElem, "conceptElem is null"); Validate.notNull(xmlNodeElem, "xmlElem is null, conceptElem=%s", conceptElem.getIdUnique()); @@ -276,20 +328,72 @@ private static void buildPhysicalModelRec(final Document doc, final FieldsAndNod System.out .println(depthStr + " " + xmlNodeElem.getTagName() + ", id=" + conceptElem.getIdUnique()); + // FIELDS. + // Put fields first because some nodes xpath predicates may depend on the presence of a field, + // like ID fields, for example something like: + // LOT-0001 + if (buildFields) { + buildFields(doc, fieldsAndNodes, conceptElem, xmlNodeElem, debug, depth, sb); + } + + // // NODES. + // for (final ConceptTreeNode conceptNode : conceptElem.getConceptNodes()) { + // The nodes may contain fields ... buildNodesAndFields(doc, fieldsAndNodes, conceptNode, xpathInst, xmlNodeElem, debug, depth, - onlyIfPriority, buildFields); + sb, buildFields); } + } - // FIELDS. + private static void buildFields(final Document doc, final FieldsAndNodes fieldsAndNodes, + final ConceptTreeNode conceptElem, final Element xmlNodeElem, final boolean debug, + final int depth, final StringBuilder sb) { + // The fields are terminal (tree leaves) and cannot contain nodes. + // But we have to deal with element attributes that must be added to some elements. + // We want the attribute information available once we reach the element that has them. + // This is done in a separator for loop as the attribute fields could appear after the fields + // that have the attributes. + final Map attributeFieldById = new HashMap<>(); + final List conceptFieldsHavingAttributes = new ArrayList<>(); + final List conceptFieldsWithoutAttributes = new ArrayList<>(); for (final ConceptTreeField conceptField : conceptElem.getConceptFields()) { - buildFields(doc, fieldsAndNodes, conceptField, xmlNodeElem, debug, depth, - onlyIfPriority, buildFields); + final String fieldId = conceptField.getFieldId(); + final JsonNode fieldMeta = fieldsAndNodes.getFieldById(fieldId); + // An attribute is an attribute of another field. + final Optional attributeOf = + JsonUtils.getTextOpt(fieldMeta, FieldsAndNodes.ATTRIBUTE_OF); + if (attributeOf.isPresent()) { + // This is an attribute field. + attributeFieldById.put(fieldId, conceptField); + } else { + if (fieldMeta.has(FieldsAndNodes.ATTRIBUTES)) { + conceptFieldsHavingAttributes.add(conceptField); + } else { + conceptFieldsWithoutAttributes.add(conceptField); + } + } + } + + // + // Build field elements and set the attributes. + // + + // First: add fields having attributes as other fields may have an xpath referring to the + // attribute (sibling fields). For example one field could be some kind of category marker. + for (final ConceptTreeField conceptField : conceptFieldsHavingAttributes) { + buildFieldElementsAndAttributes(doc, fieldsAndNodes, conceptField, xmlNodeElem, debug, + depth, sb, attributeFieldById); + } + + // Second: add fields that have no attribute. + for (final ConceptTreeField conceptField : conceptFieldsWithoutAttributes) { + buildFieldElementsAndAttributes(doc, fieldsAndNodes, conceptField, xmlNodeElem, debug, + depth, sb, attributeFieldById); } // if (debug) { - // Display the XML steps: + // Display the XML building steps: // System out is used here because it is more readable than the logger lines in the console. // This is not a replacement for logger.debug(...) // System.out.println(""); @@ -308,13 +412,11 @@ private static void buildPhysicalModelRec(final Document doc, final FieldsAndNod * @param debug Adds extra debugging info in the XML if true, for humans or unit tests, the XML * may become invalid * @param depth The current depth level passed for debugging and logging purposes - * @param onlyIfPriority Only build priority items (for xpath of other items which refer to them - * later) * @param buildFields True if fields have to be built, false otherwise */ private static boolean buildNodesAndFields(final Document doc, final FieldsAndNodes fieldsAndNodes, final ConceptTreeNode conceptNode, final XPath xpathInst, - final Element xmlNodeElem, final boolean debug, final int depth, boolean onlyIfPriority, + final Element xmlNodeElem, final boolean debug, final int depth, final StringBuilder sb, final boolean buildFields) { final String depthStr = StringUtils.leftPad(" ", depth * 4); @@ -331,19 +433,16 @@ private static boolean buildNodesAndFields(final Document doc, Element previousElem = xmlNodeElem; Element partElem = null; - // xpathRelative can contain many xml elements. We must build the hierarchy. - // TODO Use ANTLR xpath grammar later? Avoid parsing the xpath altogether? - + // xpathRelative can contain many XML elements. We must build the hierarchy. // Split the XPATH into parts. - final String[] partsArr = getXpathPartsArr(xpathRel); - final List xpathParts = new ArrayList<>(Arrays.asList(partsArr)); - // parts.remove(0); // If absolute. - // parts.remove(0); // If absolute. + final List xpathParts = getXpathParts(xpathRel); if (debug) { // System out is used here because it is more readable than the logger lines. // This is not a replacement for logger.debug(...) System.out.println(depthStr + " NODE PARTS SIZE: " + xpathParts.size()); System.out.println(depthStr + " NODE PARTS: " + listToString(xpathParts)); + + sb.append(depthStr).append("nodeId=").append(nodeId).append('\n'); } // In SDK 1.9: @@ -355,8 +454,7 @@ private static boolean buildNodesAndFields(final Document doc, for (final String xpathPart : xpathParts) { Validate.notBlank(xpathPart, "partXpath is blank for nodeId=%s, xmlNodeElem=%s", nodeId, xmlNodeElem); - final PhysicalXpathPart px = handleXpathPart(xpathPart); - final Optional schemeNameOpt = px.getSchemeNameOpt(); + final PhysicalXpathPart px = buildXpathPart(xpathPart); final String xpathExpr = px.getXpathExpr(); final String tag = px.getTagOrAttribute(); if (debug) { @@ -365,50 +463,53 @@ private static boolean buildNodesAndFields(final Document doc, System.out.println(depthStr + " tag=" + tag); System.out.println(depthStr + " xmlTag=" + xmlNodeElem.getTagName()); } - - // Find existing elements in the context of the previous element. - final NodeList foundElements; - if (previousElem.getTagName().equals(tag) && xpathExpr.equals(tag)) { // Sometimes the xpath absolute part already matches the previous element. // If there is no special xpath expression, just skip the part. // This avoids nesting of the same .../tag/tag/... - // TODO this may be fixed by TEDEFO-1466 + // This may be fixed by ticket TEDEFO-1466, but the problem could come back. + logger.warn("Same tag, skipping tag: {}", tag); continue; // Skip this tag. } - foundElements = XmlUtils.evaluateXpathAsNodeList(xpathInst, previousElem, xpathExpr, nodeId); - - if (foundElements.getLength() > 0) { - assert foundElements.getLength() == 1; - // TODO investigate what should be done if more than one is present!? - - // Node is a w3c dom node, nothing to do with the SDK node. - final Node xmlNode = foundElements.item(0); - System.out.println(depthStr + " " + "Found elements: " + foundElements.getLength()); - if (Node.ELEMENT_NODE == xmlNode.getNodeType()) { - // An existing element was found, reuse it. - partElem = (Element) xmlNode; - } else { - throw new RuntimeException(String.format("NodeType=%s not an Element", xmlNode)); - } - } else { + if (nodeMetaRepeatable) { // Create an XML element for the node. if (debug) { final String msg = String.format("%s, xml=%s", nodeId, tag); System.out.println(depthStr + " " + msg); } - partElem = createElemXml(doc, tag); - } + partElem = createElemXml(doc, tag, debug, depthStr, sb, "node"); - previousElem.appendChild(partElem); // SIDE-EFFECT! Adding item to the tree. + } else { + // Find existing elements in the context of the previous element. + final NodeList foundElements = + XmlUtils.evaluateXpathAsNodeList(xpathInst, previousElem, xpathExpr, nodeId); + if (foundElements.getLength() > 0) { + Validate.isTrue(foundElements.getLength() == 1, + "Found more than one element: {}, nodeId={}, xpathExpr={}", foundElements, nodeId, + xpathExpr); + + // Node is a w3c dom node, nothing to do with the SDK node. + final Node xmlNode = foundElements.item(0); + System.out.println(depthStr + " " + "Found elements: " + foundElements.getLength()); + if (Node.ELEMENT_NODE == xmlNode.getNodeType()) { + // An existing element was found, reuse it. + partElem = (Element) xmlNode; + } else { + throw new RuntimeException(String.format("NodeType=%s not an Element", xmlNode)); + } - if (schemeNameOpt.isPresent()) { - final String schemeName = schemeNameOpt.get(); - final String msg = String.format("%s=%s", XML_ATTR_SCHEME_NAME, schemeName); - System.out.println(depthStr + " " + msg); - partElem.setAttribute(XML_ATTR_SCHEME_NAME, schemeName); // SIDE-EFFECT! + } else { + // Create an XML element for the node. + if (debug) { + final String msg = String.format("%s, xml=%s", nodeId, tag); + System.out.println(depthStr + " " + msg); + } + partElem = createElemXml(doc, tag, debug, depthStr, sb, "node"); + } } + + previousElem.appendChild(partElem); // SIDE-EFFECT! Adding item to the tree. previousElem = partElem; } // End of for loop on parts of relative xpath. @@ -419,59 +520,67 @@ private static boolean buildNodesAndFields(final Document doc, // "id" : "ND-RegistrarAddress" // "xpathRelative" : "cac:CorporateRegistrationScheme/cac:JurisdictionRegionAddress" // The element nodeElem is cac:JurisdictionRegionAddress, so it is the node. - final Element nodeElem = partElem; - Validate.notNull(nodeElem, "partElem is null, conceptElem=%s", conceptNode.getIdUnique()); + final Element lastNodeElem = partElem; + Validate.notNull(lastNodeElem, "partElem is null, conceptElem=%s", conceptNode.getIdUnique()); // This could make the XML invalid, this is meant to be read by humans. if (debug) { - nodeElem.setAttribute(XML_ATTR_EDITOR_NODE_ID, nodeId); // SIDE-EFFECT! - - nodeElem.setAttribute(XML_ATTR_EDITOR_COUNTER_SELF, + lastNodeElem.setAttribute(XML_ATTR_EDITOR_NODE_ID, nodeId); // SIDE-EFFECT! + lastNodeElem.setAttribute(XML_ATTR_EDITOR_COUNTER_SELF, Integer.toString(conceptNode.getCounter())); // SIDE-EFFECT! } // Build child nodes recursively. - buildPhysicalModelRec(doc, fieldsAndNodes, conceptNode, nodeElem, debug, buildFields, depth + 1, - onlyIfPriority, xpathInst); + buildPhysicalModelRec(doc, fieldsAndNodes, conceptNode, lastNodeElem, debug, buildFields, + depth + 1, sb, xpathInst); return nodeMetaRepeatable; } /** - * Builds the fields, some fields have nodes in their xpath, those will also be built. As a - * side-effect the doc and the passed xml element will be modified. + * Builds the fields XML, some fields have XML elements in their xpath, those will also be built. + * As a side-effect the XML document and the passed XML element will be modified. * * @param doc The XML document, modified as a SIDE-EFFECT! * @param xmlNodeElem current XML element (modified as a SIDE-EFFECT!) * @param debug special debug mode for humans and unit tests (XML may be invalid) * @param depth The current depth level passed for debugging and logging purposes - * @param onlyIfPriority add only elements that have priority - * @param buildFields If false it will abort (only exists to simplify the code elsewhere) + * @param attributeFieldById All attribute fields by id */ - private static void buildFields(final Document doc, final FieldsAndNodes fieldsAndNodes, + private static void buildFieldElementsAndAttributes(final Document doc, + final FieldsAndNodes fieldsAndNodes, final ConceptTreeField conceptField, final Element xmlNodeElem, - final boolean debug, final int depth, final boolean onlyIfPriority, - final boolean buildFields) { + final boolean debug, final int depth, final StringBuilder sb, + final Map attributeFieldById) { - if (!buildFields) { - return; - } final String depthStr = StringUtils.leftPad(" ", depth * 4); - - // logger.debug("xmlEleme=" + EditorXmlUtils.getNodePath(xmlNodeElem)); - - final String value = conceptField.getValue(); + final String fieldValue = conceptField.getValue(); final String fieldId = conceptField.getFieldId(); + logger.debug("PM fieldId={}", fieldId); if (debug) { System.out.println(""); System.out.println(depthStr + " fieldId=" + fieldId); + + sb.append(depthStr).append("fieldId=").append(fieldId).append('\n'); } // Get the field meta-data from the SDK. final JsonNode fieldMeta = fieldsAndNodes.getFieldById(fieldId); Validate.notNull(fieldMeta, "fieldMeta null for fieldId=%s", fieldId); + final Optional attributeOf = + JsonUtils.getTextOpt(fieldMeta, FieldsAndNodes.ATTRIBUTE_OF); + final boolean isAttribute = attributeOf.isPresent(); + if (isAttribute) { + // Skip. Attributes are used later in this code from the field on which they are attached to. + // Starting from "attributes" and not from "attributeOf". + throw new RuntimeException(String + .format("Attribute fields should not reach this part of the code, fieldId=%s", fieldId)); + } + final Map attributeNameAndValueMap = + determineAttributes(fieldsAndNodes, attributeFieldById, fieldId, fieldMeta, fieldValue); + // IMPORTANT: !!! The relative xpath of fields can contain intermediary xml elements !!! // Example: "cac:PayerParty/cac:PartyIdentification/cbc:ID" contains more than just the field. // These intermediary elements are very simple items and have no nodeId. @@ -484,134 +593,198 @@ private static void buildFields(final Document doc, final FieldsAndNodes fieldsA Element previousElem = xmlNodeElem; Element partElem = null; - // TODO Use ANTLR xpath grammar later. - final String[] partsArr = getXpathPartsArr(xpathRel); - final List parts = new ArrayList<>(Arrays.asList(partsArr)); + final List parts = getXpathParts(xpathRel); if (debug) { System.out.println(depthStr + " FIELD PARTS SIZE: " + parts.size()); System.out.println(depthStr + " FIELD PARTS: " + listToString(parts)); } - final String attrTemp = "temp"; + // final String attrTemp = "temp"; for (final String partXpath : parts) { - final PhysicalXpathPart px = handleXpathPart(partXpath); - final Optional schemeNameOpt = px.getSchemeNameOpt(); + final PhysicalXpathPart px = buildXpathPart(partXpath); + // final String xpathExpr = px.getXpathExpr(); final String tagOrAttr = px.getTagOrAttribute(); - // In this case the field is an attribute of a field in the XML, technically this makes a - // difference and we have to handle this with specific code. - // Example: "@listName" - // IDEA: "attribute" : "listName" in the fields.json for fields at are attributes in the - // XML. - final boolean isAttribute = tagOrAttr.startsWith("@") && tagOrAttr.length() > 1; - - // NOTE: for the field relative xpath we want to build the all the elements (no reuse). - if (isAttribute) { - // Set attribute on previous element. - // Example: - // @listName or @currencyID - // In the case we cannot create a new XML element. - // We have to add this attribute to the previous element. - logger.debug(depthStr + " Creating attribute=" + tagOrAttr); - previousElem.setAttribute(tagOrAttr.substring(1), value); // SIDE-EFFECT! - // partElem = ... NO we do not want to reassign the partElem. This ensures that after we - // exit the loop the partElem still points to the last XML element. - // We also cannot set an attribute on an attribute! - } else { - // Create an XML element. - logger.debug(depthStr + " Creating tag=" + tagOrAttr); - partElem = createElemXml(doc, tagOrAttr); - partElem.setAttribute(attrTemp, attrTemp); - } + // Example: + // ... efbc:OverallApproximateFrameworkContractsAmount/@currencyID", the last part. + final boolean isAttributePart = tagOrAttr.startsWith("@") && tagOrAttr.length() > 1; + Validate.isTrue(!isAttributePart, "An attribute should not appear here! fieldId=%s", fieldId); - // This check is to avoid a problem with attributes. - if (!isAttribute && partElem != null) { - previousElem.appendChild(partElem); // SIDE-EFFECT! Adding item to the tree. - - if (schemeNameOpt.isPresent()) { - partElem.setAttribute(XML_ATTR_SCHEME_NAME, schemeNameOpt.get()); - } - previousElem = partElem; - } + // Create an XML element. + partElem = createElemXml(doc, tagOrAttr, debug, depthStr, sb, "field"); + // partElem.setAttribute(attrTemp, attrTemp); + previousElem.appendChild(partElem); // SIDE-EFFECT! Adding item to the XML doc tree. + previousElem = partElem; } // End of for loop on parts of relative xpath. - // We arrived at the end of the relative xpath. - // By design of the above algorithm the last element is always a leaf: the current field. - final Element fieldElem = partElem != null ? partElem : previousElem; + // We arrived at the end of the relative xpath of the field. + // By design of the above algorithm the last element is always a leaf: the current field element + final Element lastFieldElem = partElem; + Validate.notNull(lastFieldElem, "fieldElem is null for fieldId=%s, xpathRel=%s", fieldId, + xpathRel); + + // Set the attributes of this element. + for (final Entry entry : attributeNameAndValueMap.entrySet()) { + + final String attributeName = entry.getKey(); + if (StringUtils.isNotBlank(lastFieldElem.getAttribute(attributeName))) { + // If the attribute had already been set this would overwrite the existing value. + // There is no case for which this is desirable. + throw new RuntimeException(String.format( + "Double set: Attribute already set, attributeName=%s, fieldId=%s", attributeName, + fieldId)); + } - Validate.notNull(fieldElem, "fieldElem is null for fieldId=%s, xpathRel=%s", fieldId, xpathRel); + final String attributeValue; + if (XML_ATTR_LIST_NAME.equals("attributeName") + && lastFieldElem.getNodeName().equals("cbc:CapabilityTypeCode")) { + // HARDCODED: This will be fixed, probably in SDK 1.9 or SDK 1.10. + attributeValue = "sector"; + } else { + attributeValue = entry.getValue(); + } + lastFieldElem.setAttribute(attributeName, attributeValue); // SIDE-EFFECT! + } if (debug) { - // This could make the XML invalid, this is meant to be read by humans. + // This makes the XML invalid, it is meant to be read by humans to help understand the XML. // These attributes are also useful in unit tests for easy checking of field by id. - fieldElem.setAttribute(XML_ATTR_EDITOR_FIELD_ID, fieldId); + // This keeps the original visual / conceptual fieldId in the physical model. + // Another concept could be to set XML comments but this can also be a problem in unit tests. + lastFieldElem.setAttribute(XML_ATTR_EDITOR_FIELD_ID, fieldId); - fieldElem.setAttribute(XML_ATTR_EDITOR_COUNTER_SELF, - Integer.toString(conceptField.getCounter())); + final String counter = Integer.toString(conceptField.getCounter()); + lastFieldElem.setAttribute(XML_ATTR_EDITOR_COUNTER_SELF, counter); } - if (onlyIfPriority && StringUtils.isBlank(fieldElem.getAttribute(XML_ATTR_SCHEME_NAME))) { - // Remove created and appended child elements. - Element elem = fieldElem; - while (true) { - if (elem.hasAttribute(attrTemp)) { - final Node parentNode = elem.getParentNode(); - if (parentNode != null) { - parentNode.removeChild(elem); - elem = (Element) parentNode; - } else { - break; - } + // Set value of the field. + Validate.notNull(fieldValue, "value is null for fieldId=%s", fieldId, "fieldId=" + fieldId); + lastFieldElem.setTextContent(fieldValue); + } + + /** + * If the passed field has attributes the attribute name and value will be put in a map and return + * the map. + * + * @param fieldsAndNodes Holds SDK field and node metadata + * @param fieldId The field ID for the field for which the attributes have to be determined + * @param fieldMeta The SDK metadata about the field + * @param attributeFieldById Only used for reading + * @param fieldValue The value of the field + * @return A map with the determined attribute name and value for the passed field + */ + private static Map determineAttributes(final FieldsAndNodes fieldsAndNodes, + final Map attributeFieldById, final String fieldId, + final JsonNode fieldMeta, final String fieldValue) { + // Find attribute field ids of this SDK field. + final List attributeFieldIds = + JsonUtils.getListOfStrings(fieldMeta, FieldsAndNodes.ATTRIBUTES); + final Map attributeNameAndValues = + new HashMap<>(attributeFieldIds.size(), 1.0f); + for (final String attributeFieldId : attributeFieldIds) { + final ConceptTreeField conceptAttribute = attributeFieldById.get(attributeFieldId); + final JsonNode sdkAttrMeta = fieldsAndNodes.getFieldById(attributeFieldId); + final String attrName = JsonUtils.getTextStrict(sdkAttrMeta, FieldsAndNodes.ATTRIBUTE_NAME); + if (conceptAttribute != null) { + final String attributeValue = conceptAttribute.getValue(); // Value from the form. + if (StringUtils.isNotBlank(attributeValue)) { + attributeNameAndValues.put(attrName, attributeValue); } else { - break; + // It does not make sense for the value to be blank, but it can happen if there is no + // front-end validation and we want an empty form to be processed, otherwise we would have + // to only pass a fully filled and valid form, which is not easy in the editor demo. + // We want to set the attributes as those are used in the xpath expressions which are used + // to locate elements. + inferAttribute(sdkAttrMeta, attributeNameAndValues, attrName, fieldMeta, fieldValue); } + } else { + inferAttribute(sdkAttrMeta, attributeNameAndValues, attrName, fieldMeta, fieldValue); } - return; // Skip, it will be added later. + } + return attributeNameAndValues; + } + /** + * Some attributes can be determined automatically. They do not need to be put in the forms (or + * they could but would need to be hidden). + * + * @param sdkAttrMeta SDK metadata about the attribute field + * @param fieldMeta SDK metadata about the field having the attribute + * @param fieldValue The value of the field having the attributes + */ + private static void inferAttribute(final JsonNode sdkAttrMeta, + final Map attributeNameAndValues, final String attributeName, + final JsonNode fieldMeta, final String fieldValue) { + final Optional presetValueOpt = + JsonUtils.getTextOpt(sdkAttrMeta, FieldsAndNodes.PRESET_VALUE); + if (presetValueOpt.isPresent()) { + // For example: + // "id" : "OPP-070-notice-List", + // "presetValue" : "notice-subtype", + final String presetValue = presetValueOpt.get(); + attributeNameAndValues.put(attributeName, presetValue); } else { - // Remove temporary attribute. - Element elem = fieldElem; - while (true) { - if (elem.hasAttribute(attrTemp)) { - final Node parentNode = elem.getParentNode(); - if (parentNode != null) { - elem.removeAttribute(attrTemp); - elem = (Element) parentNode; - } else { - break; + logger.info("No presetValue: infer attribute? fieldId={}, fieldValue={}, attrName={}", + JsonUtils.getTextStrict(sdkAttrMeta, FieldsAndNodes.ID), fieldValue, + attributeName); + if ("id-ref".equals(JsonUtils.getTextStrict(fieldMeta, FieldsAndNodes.FIELD_TYPE)) + && StringUtils.isNotBlank(fieldValue) + && XML_ATTR_SCHEME_NAME.equals(attributeName)) { + // Example: + // In some cases for id ref there are two choices, for example ORG or TPO, + // the user selects one either ORG or TPO. + // Assuming the value in the field is "ORG-0001" or "TPO-0001", we want "ORG" or "TPO". + final int indexOfDash = fieldValue.indexOf('-'); + if (indexOfDash > 0) { + // In the database we have the "identifier_scheme". + final String idPrefix = fieldValue.substring(0, indexOfDash); + logger.info("idPrefix={}", idPrefix); + final String attrValue; + // HARDCODED until we have that mapping in the SDK. + switch (idPrefix) { + case "CON": + attrValue = "contract"; + break; + case "GLO": + attrValue = "LotsGroup"; + break; + case "LOT": + attrValue = "Lot"; + break; + case "ORG": + attrValue = "organization"; + break; + case "PAR": + attrValue = "Part"; + break; + case "RES": + attrValue = "result"; + break; + case "TEN": + attrValue = "tender"; + break; + case "TPA": + attrValue = "tendering-party"; + break; + case "TPO": + attrValue = "touchpoint"; + break; + case "UBO": + attrValue = "ubo"; + break; + default: + throw new RuntimeException(String.format("Unknown id pattern: %s", idPrefix)); } - } else { - break; + attributeNameAndValues.put(attributeName, attrValue); } } } - - // Set value of the field. - Validate.notNull(value, "value is null for fieldId=%s", fieldId, "fieldId=" + fieldId); - fieldElem.setTextContent(value); - - final String fieldType = JsonUtils.getTextStrict(fieldMeta, FIELD_TYPE); - if (FIELD_TYPE_CODE.equals(fieldType)) { - - // Find the SDK codelist identifier. - final JsonNode codelistValue = - FieldsAndNodes.getFieldPropertyValue(fieldMeta, FIELD_CODE_LIST); - String codelistName = JsonUtils.getTextStrict(codelistValue, "id", "fieldId=" + fieldId); - if (ConceptualModel.FIELD_SECTOR_OF_ACTIVITY.equals(fieldId)) { - // TODO sector, temporary hardcoded fix here, this information should be provided in the - // SDK. Maybe via a special key/value. - codelistName = "sector"; - } - - // Convention: in the XML the codelist is set in the listName attribute. - fieldElem.setAttribute(XML_ATTR_LIST_NAME, codelistName); - } } /** - * Get information about the document type from the SDK. + * Gets information about the document type from the SDK. * * @param noticeInfoBySubtype Map with info about notice metadata by notice sub type * @param documentInfoByType Map with info about document metadata by document type @@ -664,9 +837,9 @@ public static XPath setXmlNamespaces(final DocumentTypeInfo docTypeInfo, /** * @param xpath A valid xpath string - * @return The xpath string split by slash but with predicates ignored. + * @return The xpath string split by slash, the predicates are present in the output */ - private static String[] getXpathPartsArr(final String xpath) { + public static List getXpathParts(final String xpath) { final StringBuilder sb = new StringBuilder(xpath.length()); int stacked = 0; for (int i = 0; i < xpath.length(); i++) { @@ -677,90 +850,108 @@ private static String[] getXpathPartsArr(final String xpath) { stacked--; Validate.isTrue(stacked >= 0, "stacked is < 0 for %s", xpath); } + // If we are inside of a predicate, replace '/' by a temporary replacement. sb.append(ch == '/' && stacked > 0 ? XPATH_TEMP_REPLACEMENT : ch); } - return sb.toString().split("/"); - } - @edu.umd.cs.findbugs.annotations.SuppressFBWarnings( - value = "UCPM_USE_CHARACTER_PARAMETERIZED_METHOD", - justification = "OK here, used in other places as a string") - private static PhysicalXpathPart handleXpathPart(final String partParam) { - Validate.notBlank(partParam, "partParam is blank"); + // The predicates are not split thanks to usage of the temporary replacement. + final String originalRegex = "/"; + final char originalChar = '/'; + final String[] split = sb.toString().split(originalRegex); - final Optional schemeNameOpt; + // Inside each part, put back the original character. + return Arrays.stream(split) + .map(item -> item.replace(XPATH_TEMP_REPLACEMENT, originalChar)) + .collect(Collectors.toList()); + } - // NOTE: ideally we would want to fully avoid using xpath. - // NOTE: in the future the SDK will provide the schemeName separately for convenience! - String tagOrAttr = partParam; - if (tagOrAttr.contains("@schemeName='")) { - // Normalize the string before we start parsing it. - tagOrAttr = tagOrAttr.replace("@schemeName='", "@schemeName = '"); + /** + * @param xpath A valid xpath string + * @return The xpath string split by slash, the predicates are removed before the split + */ + public static List getXpathPartsWithoutPredicates(final String xpath) { + final StringBuilder sb = new StringBuilder(xpath.length()); + + int stacked = 0; + for (int i = 0; i < xpath.length(); i++) { + final char ch = xpath.charAt(i); + if (ch == XPATH_TEMP_REPLACEMENT) { + throw new RuntimeException(String.format("Found temp replacement character: %s in %s", + XPATH_TEMP_REPLACEMENT, xpath)); + } + if (ch == '[') { + stacked++; + } else if (ch == ']') { + stacked--; + Validate.isTrue(stacked >= 0, "stacked is < 0 for %s", xpath); + } + // If we are inside of a predicate, replace '/' by a temporary replacement. + if (stacked == 0 && ch != ']') { + sb.append(ch); + } } - if (tagOrAttr.contains("[not(@schemeName = 'EU')]")) { - // HARDCODED - // TODO This is a TEMPORARY FIX until we have a proper solution inside of the SDK. National is - // only indirectly described by saying not EU, but the text itself is not given. - // NOTE: SDK 1.9 will be the solution to this. + // The predicates are not split thanks to usage of the temporary replacement. + final String originalRegex = "/"; + final char originalChar = '/'; + final String[] split = sb.toString().split(originalRegex); - // Example: - // "xpathAbsolute" : "/*/cac:BusinessParty/cac:PartyLegalEntity/cbc:CompanyID[@schemeName = - // 'EU']", + // Inside each part, put back the original character. + return Arrays.stream(split) + .map(item -> item.replace(XPATH_TEMP_REPLACEMENT, originalChar)) + .collect(Collectors.toList()); + } - tagOrAttr = - tagOrAttr.replace("[not(@schemeName = 'EU')]", "[@schemeName = '" + NATIONAL + "']"); - } + @edu.umd.cs.findbugs.annotations.SuppressFBWarnings( + value = "UCPM_USE_CHARACTER_PARAMETERIZED_METHOD", + justification = "OK here, used in other places as a string") + private static PhysicalXpathPart buildXpathPart(final String xpathPart) { + Validate.notBlank(xpathPart, "partParam is blank"); - if (tagOrAttr.contains("[@schemeName = '")) { - final int indexOfSchemeName = tagOrAttr.indexOf("[@schemeName = '"); - String schemeName = tagOrAttr.substring(indexOfSchemeName + "[@schemeName = '".length()); - // Remove the '] - schemeName = schemeName.substring(0, schemeName.length() - "']".length()); - Validate.notBlank(schemeName, "schemeName is blank for %s", tagOrAttr); - tagOrAttr = tagOrAttr.substring(0, indexOfSchemeName); - schemeNameOpt = Optional.of(schemeName); - } else { - schemeNameOpt = Optional.empty(); - } + // NOTE: ideally we would want to avoid parsing xpath as much as possible. + // Since SDK 1.9 the attributes are provided via fields "attributes". + String tagName = xpathPart; // We want to remove the predicate as we only want the name. - if (tagOrAttr.contains("[")) { - // TEMPORARY FIX. - // Ignore predicate with negation as it is not useful for XML generation. - // Example: - // "xpathAbsolute" : - // "/*/cac:BusinessParty/cac:PartyLegalEntity[not(cbc:CompanyID/@schemeName = - // 'EU')]/cbc:RegistrationName", - tagOrAttr = tagOrAttr.substring(0, tagOrAttr.indexOf('[')); + if (tagName.indexOf('[') > 0) { + // Example: "cbc:somename[xyz]", we want to only keep "cbc:somename" + tagName = tagName.substring(0, tagName.indexOf('[')); } - if (tagOrAttr.contains(XPATH_TEMP_REPLACEMENT)) { - tagOrAttr = tagOrAttr.substring(0, tagOrAttr.indexOf(XPATH_TEMP_REPLACEMENT)); + if (tagName.indexOf(XPATH_TEMP_REPLACEMENT) >= 0) { + tagName = tagName.substring(0, tagName.indexOf(XPATH_TEMP_REPLACEMENT)); } // For the xpath expression keep the original param, only do the replacement. - final String xpathExpr = partParam.replaceAll(XPATH_TEMP_REPLACEMENT, "/"); - - Validate.notBlank(xpathExpr, "xpathExpr is blank for tag=%s, partParam=%s", tagOrAttr, - partParam); - return new PhysicalXpathPart(xpathExpr, tagOrAttr, schemeNameOpt); + final String xpathExpr = xpathPart.replace(XPATH_TEMP_REPLACEMENT, '/'); + Validate.notBlank(xpathExpr, "xpathExpr is blank for tag=%s, partParam=%s", tagName, + xpathExpr); + return new PhysicalXpathPart(xpathExpr, tagName); } /** * Builds a W3C DOM element. * * @param tagName The XML element tag name + * @param type * * @return A W3C DOM element (note that it is not attached to the DOM yet) */ - private static final Element createElemXml(final Document doc, final String tagName) { + private static final Element createElemXml(final Document doc, final String tagName, + final boolean debug, final String depthStr, final StringBuilder sb, final String type) { // This removes the xmlns="" that Saxon adds. try { if (tagName.startsWith("@")) { throw new RuntimeException( String.format("Expecting a tag but this is an attribute: %s", tagName)); } + if (debug) { + sb.append(depthStr).append(" creating: ") + .append(tagName) + .append(" with type=").append(type) + .append('\n'); + } + logger.debug("Creating element: {} for {}", tagName, type); return doc.createElementNS("", tagName); } catch (org.w3c.dom.DOMException ex) { logger.error("Problem creating element with tagName={}", tagName); diff --git a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/PhysicalXpathPart.java b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/PhysicalXpathPart.java index e0d0c97..a1176a4 100644 --- a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/PhysicalXpathPart.java +++ b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/PhysicalXpathPart.java @@ -1,19 +1,14 @@ package eu.europa.ted.eforms.noticeeditor.helper.notice; -import java.util.Optional; - /** * Holds data about an xpath fragment. */ public class PhysicalXpathPart { private final String xpathExpr; - private final Optional schemeNameOpt; private final String tagOrAttr; - public PhysicalXpathPart(final String xpathExpr, final String tagOrAttr, - final Optional schemeNameOpt) { + public PhysicalXpathPart(final String xpathExpr, final String tagOrAttr) { this.xpathExpr = xpathExpr; - this.schemeNameOpt = schemeNameOpt; this.tagOrAttr = tagOrAttr; } @@ -31,10 +26,4 @@ public String getXpathExpr() { return xpathExpr; } - /** - * @return An optional extracted schemeName, use it if if present - */ - public Optional getSchemeNameOpt() { - return schemeNameOpt; - } } diff --git a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/VisualModel.java b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/VisualModel.java index 52dc15e..d40b819 100644 --- a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/VisualModel.java +++ b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/notice/VisualModel.java @@ -5,6 +5,7 @@ import java.nio.file.Path; import java.util.Optional; import java.util.UUID; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Validate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,16 +19,18 @@ import eu.europa.ted.eforms.noticeeditor.util.JsonUtils; /** - * Visual model (VM). + *

The Visual Model (VM)

* *

* Wrapper around the JSON representation of the form data. Vis is be used as a shorthand for * visual. *

+ * *

* NOTE: the form data hierarchy is supposed to follow the SDK notice-types definitions (.json) * hierarchy for the given SDK and notice sub type. *

+ * *

* NOTE: the Jackson xyzNode objects are not related to the SDK node concept, it is just that the * term "node" is commonly used for items of a tree (tree nodes) and that JSON data is hierarchical. @@ -37,6 +40,11 @@ public class VisualModel { private static final Logger logger = LoggerFactory.getLogger(VisualModel.class); + /** + * This is used in the front-end, where the visual model is created. + */ + private static final String NOTICE_METADATA = "notice-metadata"; + public static final String VIS_SDK_VERSION = "sdkVersion"; public static final String VIS_NOTICE_UUID = "noticeUuid"; private static final String VIS_NOTICE_SUB_TYPE = "noticeSubType"; @@ -59,26 +67,37 @@ public class VisualModel { */ private final JsonNode visRoot; - @Override - public String toString() { - try { - return JsonUtils.getStandardJacksonObjectMapper().writeValueAsString(visRoot); - } catch (JsonProcessingException ex) { - throw new RuntimeException(ex); - } - } + /** + * This can be used for debugging, setting values only on items of interest + */ + private final boolean skipIfNoValue; /** * @param visRoot The visual model as JSON, usually set from a user interface. */ - public VisualModel(final JsonNode visRoot) { + public VisualModel(final JsonNode visRoot, final boolean skipIfNoValue) { final String rootNodeId = JsonUtils.getTextStrict(visRoot, VIS_NODE_ID); final String expected = ConceptualModel.ND_ROOT; Validate.isTrue(expected.equals(rootNodeId), "Visual model root must be %s", expected); this.visRoot = visRoot; + this.skipIfNoValue = skipIfNoValue; getNoticeSubType(); // This must not crash. } + public VisualModel(final JsonNode visRoot) { + this(visRoot, false); + } + + @Override + public String toString() { + try { + return JsonUtils.getStandardJacksonObjectMapper().writerWithDefaultPrettyPrinter() + .writeValueAsString(visRoot); + } catch (JsonProcessingException ex) { + throw new RuntimeException(ex); + } + } + private String getNoticeSubType() { return JsonUtils.getTextStrict(visRoot, VIS_NOTICE_SUB_TYPE); } @@ -149,7 +168,7 @@ public static ArrayNode setupVisualRootForTest(final ObjectMapper mapper, final ObjectNode metadata = mapper.createObjectNode(); visRootChildren.add(metadata); putGroupDef(metadata); - metadata.put(VIS_CONTENT_ID, "notice-metadata"); + metadata.put(VIS_CONTENT_ID, NOTICE_METADATA); final ArrayNode metadataChildren = metadata.putArray(VIS_CHILDREN); // Notice sub type. @@ -182,19 +201,51 @@ public static ArrayNode setupVisualRootForTest(final ObjectMapper mapper, * Build the conceptual model from the visual model. * * @param fieldsAndNodes Field and node metadata + * * @return The conceptual model for this visual model */ - public ConceptualModel toConceptualModel(final FieldsAndNodes fieldsAndNodes) { + public ConceptualModel toConceptualModel(final FieldsAndNodes fieldsAndNodes, + final boolean debug) { logger.info("Attempting to build the conceptual model from the visual model."); + // This fake top level node is used to simplify the algorithm, as we always want to have a + // parent to attach to but for the root we have none. + final ConceptTreeNode fakeConceptRoot = + new ConceptTreeNode("fake_root", "ND-Fake-Root", + 1, false); + + final StringBuilder sb = new StringBuilder(512); + // This is located in this class as most of the code is about reading the visual model. - final Optional conceptItemOpt = - parseVisualModelRec(fieldsAndNodes, visRoot, null); - if (!conceptItemOpt.isPresent()) { - throw new RuntimeException("Expecting concept item at root level."); + parseVisualModelRec(fieldsAndNodes, visRoot, fakeConceptRoot, skipIfNoValue, sb); + + if (debug) { + final Path path = Path.of("target", "debug"); + try { + Files.createDirectories(path); + JavaTools.writeTextFile(path.resolve("visual-model.json"), this.toString()); + } catch (IOException e) { + throw new RuntimeException(e); + } } - final ConceptTreeNode rootNode = (ConceptTreeNode) conceptItemOpt.get(); - return new ConceptualModel(rootNode, fieldsAndNodes.getSdkVersion()); + + if (debug) { + final Path path = Path.of("target", "debug"); + try { + Files.createDirectories(path); + JavaTools.writeTextFile(path.resolve("conceptual-model.txt"), sb.toString()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + Validate.isTrue(1 == fakeConceptRoot.getConceptNodes().size(), "Expecting one element!"); + final ConceptTreeNode conceptRoot = fakeConceptRoot.getConceptNodes().get(0); + + Validate.notEmpty(conceptRoot.getConceptNodes(), "Concept nodes list is empty"); + Validate.notEmpty(conceptRoot.getConceptFields(), "Concept fields list is empty"); + + return new ConceptualModel(conceptRoot, fieldsAndNodes.getSdkVersion()); } /** @@ -204,86 +255,223 @@ public ConceptualModel toConceptualModel(final FieldsAndNodes fieldsAndNodes) { * @param fieldsAndNodes SDK meta info * @param closestParentNode This is the closest parent node we have in the model * @param cn The current conceptual node + * @param fieldOrNodeId The id of the content for which this was started + * @return the passed conceptual node (cn), or a parent cn that was added around it */ - private static void addIntermediaryNonRepeatingNodesRec(final FieldsAndNodes fieldsAndNodes, - final ConceptTreeNode closestParentNode, final ConceptTreeNode cn) { + private static ConceptTreeNode addIntermediaryNonRepeatingNodesRec( + final FieldsAndNodes fieldsAndNodes, + final ConceptTreeNode closestParentNode, final ConceptTreeNode cn, + final String fieldOrNodeId, final StringBuilder sb) { - if (closestParentNode.getNodeId().equals(cn.getNodeId())) { + final String currentNodeId = cn.getNodeId(); + if (closestParentNode.getNodeId().equals(currentNodeId)) { // cn is the closest parent, stop. - return; + return cn; } - if (ConceptualModel.ND_ROOT.equals(cn.getNodeId())) { - return; + if (ConceptualModel.ND_ROOT.equals(currentNodeId)) { + // Went as far as possible. + return cn; } - final JsonNode nodeMeta = fieldsAndNodes.getNodeById(cn.getNodeId()); - final String nodeParentId = + final JsonNode nodeMeta = fieldsAndNodes.getNodeById(currentNodeId); + final String cnParentIdInSdk = JsonUtils.getTextStrict(nodeMeta, FieldsAndNodes.NODE_PARENT_NODE_ID); - if (nodeParentId.equals(closestParentNode.getNodeId())) { - // The closestParent is the parent, just attach it and stop. + if (cnParentIdInSdk.equals(closestParentNode.getNodeId())) { + // The closestParent is the parent of cn (desired), just attach it and stop. // -> closestParent -> cn - closestParentNode.addConceptNode(cn, false); - return; + final boolean strict = false; + final Optional addedTo = + closestParentNode.addConceptNode(cn, strict, sb, "intermediary1"); + if (addedTo.isPresent()) { + logger.debug( + "Case A: true Added intermediary concept tree node, conceptNodeId={}, sdkId={}", + cn.getIdUnique(), fieldOrNodeId); + } else { + logger.debug( + "Case A: false Added intermediary concept tree node, conceptNodeId={}, sdkId={}", + cn.getIdUnique(), fieldOrNodeId); + } + return cn; } - final boolean isRepeatable = fieldsAndNodes.isNodeRepeatable(nodeParentId); - if (isRepeatable) { + final boolean repeatable = fieldsAndNodes.isNodeRepeatable(cnParentIdInSdk); + if (repeatable) { // The SDK says the desired parentNodeId is repeatable and is missing in the // visual model, thus we have a serious problem! final String msg = - String.format("Problem in visual node hierarchy, unexpected missing repeatable nodeId=%s", - nodeParentId); + String.format( + "Problem in visual node hierarchy, unexpected missing repeatable nodeId=%s, " + + "sdkId=%s", + cnParentIdInSdk, fieldOrNodeId); System.err.println(msg); // throw new RuntimeException(msg); } + final String uniqueId = cnParentIdInSdk + SUFFIX_GENERATED; + sb.append("Adding intermediary node=") + .append(uniqueId) + .append(" for ").append(currentNodeId) + .append('\n'); + // The parent is not the closest parent we know about and it is not repeatable. // Try to create an intermediary node in the conceptual model. // -> closestParent -> cnNew -> cn final ConceptTreeNode cnNew = - new ConceptTreeNode(nodeParentId + SUFFIX_GENERATED, nodeParentId, 1, isRepeatable); - cnNew.addConceptNode(cn, false); + new ConceptTreeNode(uniqueId, cnParentIdInSdk, 1, repeatable); + final boolean strict = false; + final Optional addedTo = cnNew.addConceptNode(cn, strict, sb, "intermediaryB"); + if (addedTo.isPresent()) { + logger.debug( + "Case B: true Added intermediary concept tree node, conceptNodeId={}, sdkId={}", + cn.getIdUnique(), fieldOrNodeId); + } else { + logger.debug( + "Case B: false Added intermediary concept tree node, conceptNodeId={}, sdkId={}", + cn.getIdUnique(), fieldOrNodeId); + } // There may be more to add, recursion: - addIntermediaryNonRepeatingNodesRec(fieldsAndNodes, closestParentNode, cnNew); + return addIntermediaryNonRepeatingNodesRec(fieldsAndNodes, closestParentNode, cnNew, + fieldOrNodeId, sb); } /** * Visit the tree of the visual model and build the visual model. Depth-first order, recursive. * * @param jsonItem The current visual json item + * @param closestParentNode It must be attached to the root directly or indirectly. + * @param sb A string builder used for debugging * @return An optional concept item, if present it is to be appended outside of the call, * otherwise no action should be taken in the caller */ - private static Optional parseVisualModelRec(final FieldsAndNodes fieldsAndNodes, - final JsonNode jsonItem, final ConceptTreeNode closestParentNode) { - Validate.notNull(jsonItem, "jsonNode is null, jsonNode=%s", jsonItem); + private static void parseVisualModelRec(final FieldsAndNodes fieldsAndNodes, + final JsonNode jsonItem, final ConceptTreeNode closestParentNode, + final boolean skipIfNoValue, final StringBuilder sb) { + Validate.notNull(closestParentNode, "closestParentNode is null, jsonItem=%s", jsonItem); + Validate.notNull(jsonItem, "jsonNode is null, closestParentNode=%s", closestParentNode); final String visContentId = getContentId(jsonItem); final String visualType = getContentType(jsonItem); // // VISUAL FIELD. + // Regular form field. // if (isField(visualType)) { - // What we call a field is some kind of form field which has a value. - final JsonNode counterJson = jsonItem.get(VIS_CONTENT_COUNT); - Validate.notNull(counterJson, "visual count is null for %s", visContentId); - final int counter = jsonItem.get(VIS_CONTENT_COUNT).asInt(-1); - return handleVisualField(fieldsAndNodes, jsonItem, closestParentNode, visContentId, counter); + sb.append('\n'); + sb.append("Field: ").append(visContentId).append('\n'); + handleVisualField(fieldsAndNodes, jsonItem, closestParentNode, skipIfNoValue, visContentId, + sb); + return; } // - // VISUAL NON-FIELD (group, ...) + // VISUAL NON-FIELD. + // For example a group. // if (isNonField(visualType)) { - return handleVisualGroup(fieldsAndNodes, jsonItem, closestParentNode, visContentId); + sb.append('\n'); + sb.append("Group: ").append(visContentId).append('\n'); + handleVisualGroup(fieldsAndNodes, jsonItem, closestParentNode, visContentId, + skipIfNoValue, sb); + return; } throw new RuntimeException(String.format("Unsupported visual type '%s'", visualType)); } + private static void handleVisualField(final FieldsAndNodes fieldsAndNodes, + final JsonNode jsonItem, final ConceptTreeNode closestParentNode, final boolean skipIfNoValue, + final String visContentId, final StringBuilder sb) { + // What we call a field is some kind of form field which has a value. The value came from an + // input, textarea, combobox, ... + final JsonNode counterJson = jsonItem.get(VIS_CONTENT_COUNT); + Validate.notNull(counterJson, "visual count is null for %s", visContentId); + final int counter = jsonItem.get(VIS_CONTENT_COUNT).asInt(-1); + + final Optional conceptFieldOpt = buildVisualField(jsonItem, visContentId, + fieldsAndNodes.getFieldType(visContentId), counter, skipIfNoValue); + + if (conceptFieldOpt.isPresent()) { + final ConceptTreeField conceptItem = conceptFieldOpt.get(); + + final String sdkId = conceptItem.getIdInSdkFieldsJson(); + final JsonNode sdkFieldMeta = fieldsAndNodes.getFieldById(sdkId); + + // We found a field. + // But is the current concept hierarchy matching the hierarchy found in the SDK fields.json? + final String sdkParentNodeId = + JsonUtils.getTextStrict(sdkFieldMeta, FieldsAndNodes.FIELD_PARENT_NODE_ID); + + addConceptualItem(fieldsAndNodes, closestParentNode, conceptItem, sdkId, sdkParentNodeId, sb); + } + } + + private static void addConceptualItem(final FieldsAndNodes fieldsAndNodes, + final ConceptTreeNode closestParentNode, final ConceptTreeItem conceptItem, + final String sdkId, final String sdkParentNodeId, final StringBuilder sb) { + if (sdkParentNodeId == null) { + // Special case for the root. + sb.append("CASE OF ROOT").append('\n'); + closestParentNode.addConceptItem(conceptItem, sb); + return; + } + if (!closestParentNode.getNodeId().equals(sdkParentNodeId)) { + // The parents do not match. + final boolean repeatable = fieldsAndNodes.isNodeRepeatable(sdkParentNodeId); + if (repeatable) { + // The SDK says the desired parentNodeId is repeatable and is missing in the visual + // model, thus we have a serious problem! + final String msg = String.format( + "Problem in visual node hierarchy, fieldId=%s is not included" + + " in the correct parent. Expecting nodeId=%s but found nodeId=%s", + sdkId, sdkParentNodeId, closestParentNode.getNodeId()); + System.err.println(msg); + // throw new RuntimeException(msg); + } + + // The SDK says the desired parent node is not repeatable or "non-repeatable". We can + // tolerate that the visual model does not point to it (or not yet) and generate it as + // this + // is not problematic in the visual model in this case. Ideally we want the full SDK node + // chain to be present in the correct order in the conceptual model. + // ND-Root -> ... -> ... -> otherConceptualNode -> The field (leaf) + final Optional cnOpt = + closestParentNode.findFirstByConceptNodeId(sdkParentNodeId); + ConceptTreeNode cn; + if (cnOpt.isPresent()) { + // Reuse existing conceptual node. + cn = cnOpt.get(); + } else { + sb.append("Concept node sdkParentNodeId=") + .append(sdkParentNodeId).append(" not present in closestParentNode=") + .append(closestParentNode) + .append('\n'); + // Create and add the missing conceptual node. + // Generate missing conceptual node. + // IDEA what if more than one intermediary nodes are missing? For now we will assume + // that this is not the case. + // ND-Root -> ... -> closestParentNode -> newConceptNode -> ... -> field + // By convention we will add a suffix to these generated concept nodes. + final String idUnique = sdkParentNodeId + SUFFIX_GENERATED; + cn = new ConceptTreeNode(idUnique, sdkParentNodeId, 1, + repeatable); + + // See unit test about filling to fully understand this. + // closestParentNode.addConceptNode(cn); // NO: there may be more items to fill in. + // cn = addIntermediaryNonRepeatingNodesRec( // NO ! + addIntermediaryNonRepeatingNodesRec(fieldsAndNodes, closestParentNode, cn, sdkId, + sb); + } + + // Always add the current item. + cn.addConceptItem(conceptItem, sb); + } else { + closestParentNode.addConceptItem(conceptItem, sb); + } + } + private static boolean isNonField(final String visualType) { return VIS_TYPE_NON_FIELD.equals(visualType); } @@ -300,15 +488,24 @@ private static String getContentId(final JsonNode jsonItem) { return JsonUtils.getTextStrict(jsonItem, VIS_CONTENT_ID); } + private static String getContentCount(final JsonNode jsonItem) { + return JsonUtils.getTextStrict(jsonItem, VIS_CONTENT_COUNT); + } + private static ArrayNode getChildren(final JsonNode item) { return (ArrayNode) item.get(VIS_CHILDREN); } - private static Optional handleVisualGroup(final FieldsAndNodes fieldsAndNodes, - final JsonNode jsonItem, final ConceptTreeNode closestParentNode, final String contentId) { + /** + * + * @param closestParentNode May be modified as a side effect. + */ + private static void handleVisualGroup(final FieldsAndNodes fieldsAndNodes, + final JsonNode visualItem, final ConceptTreeNode closestParentNode, final String contentId, + final boolean skipIfNoValue, final StringBuilder sb) { - // This is a group (with or without nodeId). - final Optional nodeIdOpt = getNodeIdOpt(jsonItem); + // This is a visual group (with or without nodeId). + final Optional visualNodeIdOpt = getNodeIdOpt(visualItem); // // GROUP WITH NO nodeId. @@ -316,170 +513,161 @@ private static Optional handleVisualGroup(final FieldsAndNodes // This group is used purely to group fields visually in the UI (visual model). // We could call this a purely visual group. // - if (nodeIdOpt.isEmpty()) { - handleGroupWithNodeId(fieldsAndNodes, jsonItem, closestParentNode); - return Optional.empty(); // Cannot return anything to append to as it was removed. + if (visualNodeIdOpt.isEmpty()) { + sb.append("Visual node id: none").append('\n'); + handleGroupWithoutNodeId(fieldsAndNodes, visualItem, closestParentNode, skipIfNoValue, sb); + } else { + sb.append("Visual node id: ").append(visualNodeIdOpt.get()).append('\n'); + // + // GROUP WITH nodeId. + // + // This is a group which references a node. + // This group must be kept in the conceptual model. + // + handleGroupWithNodeId(fieldsAndNodes, visualItem, closestParentNode, contentId, + visualNodeIdOpt.get(), skipIfNoValue, sb); } - - // - // GROUP WITH nodeId. - // - // This is a group which references a node. - // This group must be kept in the conceptual model. - // - final ConceptTreeNode conceptNode = - handleGroupWithoutNodeId(fieldsAndNodes, jsonItem, contentId, nodeIdOpt); - return Optional.of(conceptNode); } private static Optional getNodeIdOpt(final JsonNode jsonItem) { return JsonUtils.getTextOpt(jsonItem, VIS_NODE_ID); } - private static ConceptTreeNode handleGroupWithoutNodeId(final FieldsAndNodes fieldsAndNodes, - final JsonNode jsonItem, final String contentId, final Optional nodeIdOpt) { + private static void handleGroupWithNodeId(final FieldsAndNodes fieldsAndNodes, + final JsonNode visualItem, final ConceptTreeNode closestParentNode, final String visContentId, + final String sdkNodeId, final boolean skipIfNoValue, final StringBuilder sb) { + Validate.notBlank(sdkNodeId, "sdkNodeId is blank for visContentId=%s", visContentId); - final String sdkNodeId = nodeIdOpt.get(); - fieldsAndNodes.getNodeById(sdkNodeId); // Just for the checks. + final boolean repeatable = fieldsAndNodes.isNodeRepeatable(sdkNodeId); + final ConceptTreeNode conceptNodeNew = new ConceptTreeNode(visContentId, sdkNodeId, + visualItem.get(VIS_CONTENT_COUNT).asInt(-1), repeatable); - final boolean isRepeatable = fieldsAndNodes.isNodeRepeatable(nodeIdOpt.get()); - final ConceptTreeNode conceptNode = new ConceptTreeNode(contentId, sdkNodeId, - jsonItem.get(VIS_CONTENT_COUNT).asInt(-1), isRepeatable); + final JsonNode sdkNodeMeta = fieldsAndNodes.getNodeById(sdkNodeId); - // Not a leaf of the tree: recursion on children: - final JsonNode maybeNull = VisualModel.getChildren(jsonItem); + // We found a field. + // But is the current concept hierarchy matching the hierarchy found in the SDK + // fields.json? + final String sdkParentNodeId = FieldsAndNodes.ND_ROOT.equals(sdkNodeId) ? null + : JsonUtils.getTextStrict(sdkNodeMeta, FieldsAndNodes.NODE_PARENT_NODE_ID); + + // This concept node is not yet attached to anything as it was just created! + // Attach it. + addConceptualItem(fieldsAndNodes, closestParentNode, conceptNodeNew, sdkNodeId, + sdkParentNodeId, sb); + // + // Not a leaf of the tree: recursion on children: + // // The children array could be null depending on how the JSON is serialized (not present in the // JSON at all means null, or empty []). - if (maybeNull != null) { - final ArrayNode visChildren = (ArrayNode) maybeNull; - for (final JsonNode visChild : visChildren) { - final Optional itemToAppendOpt = - parseVisualModelRec(fieldsAndNodes, visChild, conceptNode); - if (itemToAppendOpt.isPresent()) { - // Append field or node. - conceptNode.addConceptItem(itemToAppendOpt.get()); - } - } + final JsonNode childrenMaybeNull = VisualModel.getChildren(visualItem); + if (childrenMaybeNull == null) { + sb.append("Found no visual child items for ").append(visContentId).append('\n'); + return; // Cannot return anything to append to. + } + final ArrayNode visChildren = (ArrayNode) childrenMaybeNull; + for (final JsonNode visChild : visChildren) { + parseVisualModelRec(fieldsAndNodes, visChild, conceptNodeNew, skipIfNoValue, sb); } - return conceptNode; } - private static void handleGroupWithNodeId(final FieldsAndNodes fieldsAndNodes, - final JsonNode jsonItem, final ConceptTreeNode closestParentNode) { + private static void handleGroupWithoutNodeId(final FieldsAndNodes fieldsAndNodes, + final JsonNode visualItem, final ConceptTreeNode closestParentNode, + final boolean skipIfNoValue, final StringBuilder sb) { // The conceptual model must ignore this group but keep the contained content. // In that case we want the children to be moved up to the nearest conceptual parent node. // This is flattening/simplifying the tree. // In other words the visual tree has extra items that the conceptual model does not need. - final JsonNode maybeNull = jsonItem.get(VIS_CHILDREN); - // Could be "null if empty" depending on how the JSON is constructed. // No children in JSON could be a value like [] or just no key value pair. // Both possibilities are tolerated. + final JsonNode maybeNull = VisualModel.getChildren(visualItem); if (maybeNull == null) { + sb.append("Visual item has no child items, cannot return anything to append to").append('\n'); return; // Cannot return anything to append to. } final ArrayNode visChildren = (ArrayNode) maybeNull; for (final JsonNode visChild : visChildren) { - final Optional itemToAppendOpt = - parseVisualModelRec(fieldsAndNodes, visChild, closestParentNode); - if (itemToAppendOpt.isPresent()) { - final ConceptTreeItem item = itemToAppendOpt.get(); - Validate.notNull(closestParentNode, "closestParentNode is null for %s", item.getIdUnique()); - closestParentNode.addConceptItem(item); - } + parseVisualModelRec(fieldsAndNodes, visChild, closestParentNode, skipIfNoValue, sb); } } - private static Optional handleVisualField(final FieldsAndNodes fieldsAndNodes, - final JsonNode jsonItem, final ConceptTreeNode closestParentNode, final String contentId, - final int counter) { + /** + * @param jsonItem The current visual json item + * @param sdkFieldId The SDK field id for the current visual json item + * @param sdkFieldType The field type as provided by the corresponding SDK + * @param counter The counter value of this field + * @param skipIfNoValue Skip if there is no value + * + * @return The concept field, it can be empty if there is no value + */ + private static Optional buildVisualField( + final JsonNode jsonItem, final String sdkFieldId, + final String sdkFieldType, final int counter, final boolean skipIfNoValue) { // This is a visual field (leaf of the tree). - // Every field points to an SDK field for the SDK metadata. - final String sdkFieldId = contentId; - final ConceptTreeField conceptField = - new ConceptTreeField(contentId, sdkFieldId, jsonItem.get(VIS_VALUE).asText(null), counter); - - final JsonNode sdkFieldMeta = fieldsAndNodes.getFieldById(sdkFieldId); - - // We found a field. - // But is the current concept hierarchy matching the hierarchy found in the SDK fields.json? - final String sdkParentNodeId = - JsonUtils.getTextStrict(sdkFieldMeta, FieldsAndNodes.FIELD_PARENT_NODE_ID); - - if (!closestParentNode.getNodeId().equals(sdkParentNodeId)) { - // The parents do not match. - - final boolean isRepeatable = fieldsAndNodes.isNodeRepeatable(sdkParentNodeId); - if (isRepeatable) { - // The SDK says the desired parentNodeId is repeatable and is missing in the visual model, - // thus we have a serious problem! - final String msg = String.format( - "Problem in visual node hierarchy, fieldId=%s is not included" - + " in the correct parent. Expecting %s but found %s", - sdkFieldId, sdkParentNodeId, closestParentNode.getNodeId()); - System.err.println(msg); - // throw new RuntimeException(msg); - } + // Every visual field points to an SDK field for the SDK metadata. + final String value = jsonItem.get(VIS_VALUE).asText(null); + if (skipIfNoValue && StringUtils.isBlank(value)) { + // This helps when debugging, ot reduce the noise created by empty fields. + return Optional.empty(); + } - // The SDK says the desired parent node is not repeatable or "non-repeatable". We can - // tolerate that the visual model does not point to it (or not yet) and generate it as this - // is not problematic in the visual model in this case. Ideally we want the full SDK node - // chain to be present in the correct order in the conceptual model. - // ND-Root -> ... -> ... -> otherConceptualNode -> The field (leaf) - final Optional cnOpt = - closestParentNode.findFirstByConceptNodeId(sdkParentNodeId); - final ConceptTreeNode cn; - if (cnOpt.isPresent()) { - // Reuse existing conceptual node. - cn = cnOpt.get(); - } else { - // Create and add the missing conceptual node. - // Generate missing conceptual node. - // IDEA what if more than one intermediary nodes are missing? For now we will assume that - // this is not the case. - // ND-Root -> ... -> closestParentNode -> newConceptNode -> ... -> field - // By convention we will add a suffix to these generated concept nodes. - cn = new ConceptTreeNode(sdkParentNodeId + SUFFIX_GENERATED, sdkParentNodeId, 1, - isRepeatable); + // + // Timezones. + // + // NOTE: the editor demo only allows to select a date or a time (no timezone field has been + // foreseen). + // In order to generate valid XML, the UTC timezone (Z) has been chosen. + // In your own applications, add the necessary work in the UI so that users may specify the + // appropriate timezone and also load it from a notice. + // The browser type="date" is nice, but has no concept of timezones. + // The browser type="datetime-local" has such concepts, but merges dates and times. + final String defaultTimezone = "Z"; + final String adaptedValue; + if (value != null && (sdkFieldType.equals("date") || sdkFieldType.equals("time")) && + !value.endsWith(defaultTimezone) + && (!value.contains("+") || StringUtils.countMatches(value, '-') < 3)) { + // This could be handled in the UI. + // Put default. + adaptedValue = value + defaultTimezone; + } else { + adaptedValue = value; + } - // See unit test about filling to fully understand this. - // closestParentNode.addConceptNode(cn); // NO: there may be more items to fill in. - addIntermediaryNonRepeatingNodesRec(fieldsAndNodes, closestParentNode, cn); - } + // It probably does not make sense to keep trailing whitespace from the UI values. + final String valueStripped = adaptedValue != null ? adaptedValue.strip() : value; - // Always add the current field. - cn.addConceptField(conceptField); + final ConceptTreeField conceptField = + new ConceptTreeField(sdkFieldId, sdkFieldId, + valueStripped, + counter); - return Optional.empty(); - } return Optional.of(conceptField); // Leaf of tree: just return. } + // private static String getUniqueIdForDotSimple( + // final String uniqueIdOfParentForDot, final JsonNode item, final Optional nodeIdOpt) { + // return getContentId(item) + (nodeIdOpt.isPresent() ? "_" + nodeIdOpt.get() : ""); + // } - /** - * This can be used for visualization of the visual model tree (as a graph). The graphviz dot text - * itself is interesting but it makes even more sense when seen inside a tool. - */ - private String toDot(final FieldsAndNodes fieldsAndNodes, final boolean includeFields) { + private static String getUniqueIdForDot( + final String uniqueIdOfParentForDot, final JsonNode item, final Optional nodeIdOpt) { + return - final StringBuilder sb = new StringBuilder(1024); - final JsonNode root = this.visRoot; - toDotRec(fieldsAndNodes, sb, root, includeFields); + // Added so to make it unique in the dot ... (but hard to read) + (StringUtils.isNotBlank(uniqueIdOfParentForDot) ? "_" + uniqueIdOfParentForDot : "") - final StringBuilder sbDot = new StringBuilder(); - final String noticeSubType = this.getNoticeSubType(); - final String title = "visual_" + noticeSubType; // - is not supported. - GraphvizDotTool.appendDiGraph(sb.toString(), sbDot, title, "Visual model of " + noticeSubType, - false, true); + + getContentId(item) - return sbDot.toString(); - } + + (nodeIdOpt.isPresent() ? "_" + nodeIdOpt.get() : "") + // We need the count to avoid merging of repeating nodes and fields. + + "_" + getContentCount(item); + } /** * Recursively create DOT format text and append it to the string builder (sb). @@ -487,11 +675,11 @@ private String toDot(final FieldsAndNodes fieldsAndNodes, final boolean includeF * @param fieldsAndNodes SDK field and node metadata * @param includeFields If true include fields in the graph, otherwise do not */ - private static void toDotRec(final FieldsAndNodes fieldsAndNodes, final StringBuilder sb, - final JsonNode item, final boolean includeFields) { + private static void toDotRec(final String uniqueIdOfParentForDot, + final FieldsAndNodes fieldsAndNodes, + final StringBuilder sb, final JsonNode item, final boolean includeFields) { final Optional nodeIdOpt = getNodeIdOpt(item); - final String idUnique = - getContentId(item) + (nodeIdOpt.isPresent() ? "_" + nodeIdOpt.get() : ""); + final String idUniqueForDot = getUniqueIdForDot(uniqueIdOfParentForDot, item, nodeIdOpt); final String edgeLabel = ""; // final String edgeLabel = nodeIdOpt.isPresent() ? nodeIdOpt.get() : idUnique; @@ -506,19 +694,36 @@ private static void toDotRec(final FieldsAndNodes fieldsAndNodes, final StringBu final String color = nodeIsRepeatable ? GraphvizDotTool.COLOR_GREEN : GraphvizDotTool.COLOR_BLACK; - final String idUniqueChild = - getContentId(child) + (childNodeIdOpt.isPresent() ? "_" + childNodeIdOpt.get() : ""); - GraphvizDotTool.appendEdge(edgeLabel, color, - - idUnique, idUniqueChild, // concept node -> child concept node + final String idUniqueChildForDot = getUniqueIdForDot(idUniqueForDot, child, childNodeIdOpt); + GraphvizDotTool.appendEdge(edgeLabel, color, + idUniqueForDot, idUniqueChildForDot, // concept node -> child concept node sb); - toDotRec(fieldsAndNodes, sb, child, includeFields); + toDotRec(idUniqueChildForDot, fieldsAndNodes, sb, child, includeFields); } } } + /** + * This can be used for visualization of the visual model tree (as a graph). The graphviz dot text + * itself is interesting but it makes even more sense when seen inside a tool. + */ + private String toDot(final FieldsAndNodes fieldsAndNodes, final boolean includeFields) { + + final StringBuilder sb = new StringBuilder(1024); + final JsonNode root = this.visRoot; + toDotRec("", fieldsAndNodes, sb, root, includeFields); + + final StringBuilder sbDot = new StringBuilder(); + final String noticeSubType = this.getNoticeSubType(); + final String title = "visual_" + noticeSubType; // - is not supported. + GraphvizDotTool.appendDiGraph(sb.toString(), sbDot, title, "Visual model of " + noticeSubType, + false, true); + + return sbDot.toString(); + } + /** * Write dot graph file for debugging purposes. */ @@ -530,7 +735,7 @@ public void writeDotFile(final FieldsAndNodes fieldsAndNodes) { // Visualizing it can help understand how it works or find problems. final boolean includeFields = true; final String dotText = this.toDot(fieldsAndNodes, includeFields); - final Path pathToFolder = Path.of("target/dot/"); + final Path pathToFolder = Path.of("target", "dot"); Files.createDirectories(pathToFolder); final Path pathToFile = pathToFolder.resolve(this.getNoticeSubType() + "-visual.dot"); JavaTools.writeTextFile(pathToFile, dotText); @@ -539,5 +744,4 @@ public void writeDotFile(final FieldsAndNodes fieldsAndNodes) { } } - } diff --git a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/validation/XsdValidator.java b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/validation/XsdValidator.java index ba6d51a..975989c 100644 --- a/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/validation/XsdValidator.java +++ b/src/main/java/eu/europa/ted/eforms/noticeeditor/helper/validation/XsdValidator.java @@ -20,7 +20,7 @@ public class XsdValidator { public static List validateXml(final String xmlAsText, final Path mainXsdPath) throws SAXException, IOException { - logger.info(String.format("Attempting to validate using schema: {}", mainXsdPath)); + logger.info("Attempting to validate using schema: {}", mainXsdPath); final XsdCustomErrorHandler xsdErrorHandler = new XsdCustomErrorHandler(); diff --git a/src/main/java/eu/europa/ted/eforms/noticeeditor/service/XmlWriteService.java b/src/main/java/eu/europa/ted/eforms/noticeeditor/service/XmlWriteService.java index ca4ef0e..bcd7ae5 100644 --- a/src/main/java/eu/europa/ted/eforms/noticeeditor/service/XmlWriteService.java +++ b/src/main/java/eu/europa/ted/eforms/noticeeditor/service/XmlWriteService.java @@ -56,10 +56,15 @@ public class XmlWriteService { * @param noticeJson The notice as JSON as built by the front-end form. * @param debug Adds special debug info to the XML, useful for humans and unit tests. Not for * production + * @param skipIfNoValue Ignore if there is no value + * @param sortXml Sort the XML elements, setting false can be used for development / debugging + * purposes */ public void saveNoticeAsXml(final Optional responseOpt, - final String noticeJson, final boolean debug) throws Exception { - final PhysicalModel physicalModel = buildPhysicalModel(noticeJson, debug); + final String noticeJson, final boolean debug, final boolean skipIfNoValue, + final boolean sortXml) throws Exception { + final PhysicalModel physicalModel = + buildPhysicalModel(noticeJson, debug, skipIfNoValue, sortXml); final UUID noticeUuid = physicalModel.getNoticeId(); final SdkVersion sdkVersion = physicalModel.getSdkVersion(); try { @@ -90,7 +95,10 @@ public void saveNoticeAsXml(final Optional responseOpt, */ public void validateUsingXsd(final Optional responseOpt, final String noticeJson, final boolean debug) throws Exception { - final PhysicalModel physicalModel = buildPhysicalModel(noticeJson, debug); + final boolean skipIfNoValue = false; + final boolean sortXml = true; + final PhysicalModel physicalModel = + buildPhysicalModel(noticeJson, debug, skipIfNoValue, sortXml); final SdkVersion sdkVersion = physicalModel.getSdkVersion(); final UUID noticeUuid = physicalModel.getNoticeId(); try { @@ -118,7 +126,8 @@ public void validateUsingXsd(final Optional responseOpt, } } - public PhysicalModel buildPhysicalModel(final String noticeJson, final boolean debug) + public PhysicalModel buildPhysicalModel(final String noticeJson, final boolean debug, + final boolean skipIfNoValue, final boolean sortXml) throws Exception { final ObjectMapper mapper = JsonUtils.getStandardJacksonObjectMapper(); final JsonNode visualRoot = mapper.readTree(noticeJson); @@ -126,7 +135,7 @@ public PhysicalModel buildPhysicalModel(final String noticeJson, final boolean d final UUID noticeUuid = parseNoticeUuid(visualRoot); try { logger.info("Attempting to transform visual model into physical model as XML."); - return buildPhysicalModel(visualRoot, sdkVersion, noticeUuid, debug); + return buildPhysicalModel(visualRoot, sdkVersion, noticeUuid, debug, skipIfNoValue, sortXml); } catch (final Exception e) { // Catch any error, log some useful context and rethrow. logger.error("Error for notice uuid={}, sdkVersion={}", noticeUuid, @@ -148,7 +157,10 @@ public void validateUsingCvs(final Optional responseOpt, final String noticeJson, final boolean debug) throws Exception { Validate.notBlank(noticeJson, "noticeJson is blank"); - final PhysicalModel physicalModel = buildPhysicalModel(noticeJson, debug); + final boolean skipIfNoValue = false; + final boolean sortXml = true; + final PhysicalModel physicalModel = + buildPhysicalModel(noticeJson, debug, skipIfNoValue, sortXml); final UUID noticeUuid = physicalModel.getNoticeId(); final SdkVersion sdkVersion = physicalModel.getSdkVersion(); @@ -173,54 +185,67 @@ public void validateUsingCvs(final Optional responseOpt, } /** + * Goes from the visual model to the physical model of a notice. The conceptual model is a hidden + * intermediary step. + * + * @param visualRoot The root of the visual model of the notice, this is the main input + * @param sdkVersion The SDK version of the notice * @param debug Adds special debug info to the XML, useful for humans and unit tests. Not for * production - * @return The physical model + * @param skipIfNoValue Skip items if there is no value, this can help with debugging + * @param sortXml Sorts the XML if true, false otherwise. This can be used for debugging or + * development purposes + * @return The physical model of the notice as the output */ private PhysicalModel buildPhysicalModel(final JsonNode visualRoot, final SdkVersion sdkVersion, - final UUID noticeUuid, final boolean debug) + final UUID noticeUuid, final boolean debug, final boolean skipIfNoValue, + final boolean sortXml) throws ParserConfigurationException, SAXException, IOException { Validate.notNull(visualRoot); Validate.notNull(noticeUuid); - final FieldsAndNodes fieldsAndNodes = readFieldsAndNodes(sdkVersion); - final VisualModel visualModel = new VisualModel(visualRoot); + final FieldsAndNodes sdkFieldsAndNodes = readSdkFieldsAndNodes(sdkVersion); + final VisualModel visualModel = new VisualModel(visualRoot, skipIfNoValue); if (debug) { - visualModel.writeDotFile(fieldsAndNodes); + visualModel.writeDotFile(sdkFieldsAndNodes); } final JsonNode noticeTypesJson = sdkService.readNoticeTypesJson(sdkVersion); - - final Map noticeInfoBySubtype = new HashMap<>(512); - { - // TODO add noticeSubTypes to the SDK constants. - // SdkResource.NOTICE_SUB_TYPES - final JsonNode noticeSubTypes = noticeTypesJson.get("noticeSubTypes"); - for (final JsonNode item : noticeSubTypes) { - // TODO add subTypeId to the SDK constants. - final String subTypeId = JsonUtils.getTextStrict(item, "subTypeId"); - noticeInfoBySubtype.put(subTypeId, item); - } - } - + final Map noticeInfoBySubtype = loadSdkNoticeTypeInfo(noticeTypesJson); final Map documentInfoByType = parseDocumentTypes(noticeTypesJson); // Go from visual model to conceptual model. - final ConceptualModel conceptModel = visualModel.toConceptualModel(fieldsAndNodes); + final ConceptualModel conceptModel = + visualModel.toConceptualModel(sdkFieldsAndNodes, debug); // Build physical model. final boolean buildFields = true; final Path sdkRootFolder = sdkService.getSdkRootFolder(); final PhysicalModel physicalModel = PhysicalModel.buildPhysicalModel(conceptModel, - fieldsAndNodes, noticeInfoBySubtype, documentInfoByType, debug, buildFields, sdkRootFolder); + sdkFieldsAndNodes, noticeInfoBySubtype, documentInfoByType, debug, buildFields, + sdkRootFolder, sortXml); + return physicalModel; } - public FieldsAndNodes readFieldsAndNodes(final SdkVersion sdkVersion) { + private static Map loadSdkNoticeTypeInfo( + final JsonNode noticeTypesJson) { + final Map noticeInfoBySubtype = new HashMap<>(512); + // TODO add "noticeSubTypes" to the SDK constants. + // SdkResource.NOTICE_SUB_TYPES + final JsonNode noticeSubTypes = noticeTypesJson.get("noticeSubTypes"); + for (final JsonNode item : noticeSubTypes) { + // TODO add "subTypeId" to the SDK constants. + final String subTypeId = JsonUtils.getTextStrict(item, "subTypeId"); + noticeInfoBySubtype.put(subTypeId, item); + } + return noticeInfoBySubtype; + } + + public FieldsAndNodes readSdkFieldsAndNodes(final SdkVersion sdkVersion) { final JsonNode fieldsJson = sdkService.readSdkFieldsJson(sdkVersion); - final FieldsAndNodes fieldsAndNodes = new FieldsAndNodes(fieldsJson, sdkVersion); - return fieldsAndNodes; + return new FieldsAndNodes(fieldsJson, sdkVersion); } public static Map parseDocumentTypes(final JsonNode noticeTypesJson) { @@ -228,7 +253,7 @@ public static Map parseDocumentTypes(final JsonNode noticeType final JsonNode documentTypes = noticeTypesJson.get(SdkConstants.NOTICE_TYPES_JSON_DOCUMENT_TYPES_KEY); for (final JsonNode item : documentTypes) { - // TODO add document type id to the SDK constants. + // TODO add document type "id" to the SDK constants. Maybe DOCUMENT_TYPE_ID. final String id = JsonUtils.getTextStrict(item, "id"); documentInfoByType.put(id, item); } @@ -346,4 +371,5 @@ private static void serveJson(final HttpServletResponse response, throw new RuntimeException("IOException writing JSON to output stream.", ex); } } + } diff --git a/src/main/java/eu/europa/ted/eforms/noticeeditor/sorting/NoticeXmlTagSorter.java b/src/main/java/eu/europa/ted/eforms/noticeeditor/sorting/NoticeXmlTagSorter.java index e1755e0..4749021 100644 --- a/src/main/java/eu/europa/ted/eforms/noticeeditor/sorting/NoticeXmlTagSorter.java +++ b/src/main/java/eu/europa/ted/eforms/noticeeditor/sorting/NoticeXmlTagSorter.java @@ -92,10 +92,18 @@ public void sortXml(final Document doc) throws SAXException, IOException { */ public void sortXml(final Element xmlRoot) throws SAXException, IOException { + final String cbcCustomizationId = PhysicalModel.CBC_CUSTOMIZATION_ID; // Compare sdkVersion of the element to the SDK version of this instance. - final String sdkVersionOfNoticeStr = - XmlUtils.getDirectChild(xmlRoot, PhysicalModel.CBC_CUSTOMIZATION_ID).getTextContent(); + final Element customizationElem = + XmlUtils.getDirectChild(xmlRoot, cbcCustomizationId); + if (customizationElem == null) { + throw new RuntimeException( + String.format( + "Failed to find field=%s, the physical model is probably not well formed. xmlRoot=%s", + cbcCustomizationId, xmlRoot)); + } + final String sdkVersionOfNoticeStr = customizationElem.getTextContent(); final SdkVersion sdkVersionOfNotice = VersionHelper.parsePrefixedSdkVersion(sdkVersionOfNoticeStr); final SdkVersion sdkVersionOfSorter = getSorterSdkVersion(); @@ -213,7 +221,6 @@ public void sortRecursive(final Element xmlRootElem, final JsonNode fieldOrNode, Validate.notNull(fieldOrNode); final String id = JsonUtils.getTextStrict(fieldOrNode, FieldsAndNodes.FIELD_OR_NODE_ID_KEY); - logger.debug("Sorting children of id={}", id); final String xpathAbsolute = JsonUtils.getTextStrict(fieldOrNode, FieldsAndNodes.XPATH_ABSOLUTE); @@ -225,6 +232,7 @@ public void sortRecursive(final Element xmlRootElem, final JsonNode fieldOrNode, if (childItems == null) { return; // Nothing to sort. } + logger.debug("Sorting children of id={}", id); // Get sort order of child items for the current node id. final List orderItemsForParent = new ArrayList<>(childItems.size()); @@ -232,7 +240,7 @@ public void sortRecursive(final Element xmlRootElem, final JsonNode fieldOrNode, final String fieldOrNodeId = JsonUtils.getTextStrict(childItem, FieldsAndNodes.FIELD_OR_NODE_ID_KEY); - logger.debug("Found child fieldOrNodeId={}", fieldOrNodeId); + logger.trace("Found child fieldOrNodeId={}", fieldOrNodeId); final String xpathRel = JsonUtils.getTextStrict(childItem, FieldsAndNodes.XPATH_RELATIVE); final List xpathRelParts = XpathUtils.getXpathParts(xpathRel); @@ -251,7 +259,7 @@ public void sortRecursive(final Element xmlRootElem, final JsonNode fieldOrNode, final OrderItem orderItem = new OrderItem(fieldOrNodeId, key, order); orderItemsForParent.add(orderItem); } else { - logger.info("parentId={}, itemId={} has no {}", id, fieldOrNodeId, + logger.warn("parentId={}, itemId={} has no {}", id, fieldOrNodeId, FieldsAndNodes.XSD_SEQUENCE_ORDER_KEY); // Ideally we want this to throw, but some tests are using dummy data that is missing the // sort order and the tests are not about the order. @@ -262,7 +270,7 @@ public void sortRecursive(final Element xmlRootElem, final JsonNode fieldOrNode, } // The order items are not ordered yet, they contain the order, and we naturally sort on it. Collections.sort(orderItemsForParent); // Relies on implementation of "Comparable". - logger.debug("orderItemsForParent=" + orderItemsForParent); + logger.trace("orderItemsForParent={}", orderItemsForParent); // // Find parent elements in the XML. diff --git a/src/main/java/eu/europa/ted/eforms/noticeeditor/util/EditorXmlUtils.java b/src/main/java/eu/europa/ted/eforms/noticeeditor/util/EditorXmlUtils.java index d724b7b..fce54a0 100644 --- a/src/main/java/eu/europa/ted/eforms/noticeeditor/util/EditorXmlUtils.java +++ b/src/main/java/eu/europa/ted/eforms/noticeeditor/util/EditorXmlUtils.java @@ -53,7 +53,6 @@ public static String asText(final Document doc, final boolean indented) { } } - public static String getNodePath(final Node node) { if (node == null) { throw new IllegalArgumentException("Node cannot be null"); diff --git a/src/main/java/eu/europa/ted/eforms/noticeeditor/util/JsonUtils.java b/src/main/java/eu/europa/ted/eforms/noticeeditor/util/JsonUtils.java index dc473c3..682b155 100644 --- a/src/main/java/eu/europa/ted/eforms/noticeeditor/util/JsonUtils.java +++ b/src/main/java/eu/europa/ted/eforms/noticeeditor/util/JsonUtils.java @@ -125,6 +125,10 @@ public static String marshall(final Object obj) throws JsonProcessingException { return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(obj); } + public static List getListOfStrings(final JsonNode jsonNode, final String key) { + return getListOfStrings(jsonNode.get(key)); + } + public static List getListOfStrings(final JsonNode jsonNode) { if (jsonNode != null && jsonNode.isArray()) { final ArrayNode arrayNode = (ArrayNode) jsonNode; diff --git a/src/main/java/eu/europa/ted/eforms/noticeeditor/util/XmlUtils.java b/src/main/java/eu/europa/ted/eforms/noticeeditor/util/XmlUtils.java index 7239fc5..010792f 100644 --- a/src/main/java/eu/europa/ted/eforms/noticeeditor/util/XmlUtils.java +++ b/src/main/java/eu/europa/ted/eforms/noticeeditor/util/XmlUtils.java @@ -166,6 +166,18 @@ public static List evaluateXpathAsElemList(final XPath xpathInst, return elemList; } + public static List evaluateXpathAsListOfNode(final XPath xpathInst, + final Object contextElem, final String xpathExpr, String idForError) { + final NodeList elemsFound = + evaluateXpathAsNodeList(xpathInst, contextElem, xpathExpr, idForError); + final List elemList = new ArrayList<>(elemsFound.getLength()); + for (int i = 0; i < elemsFound.getLength(); i++) { + final Node xmlNode = elemsFound.item(i); + elemList.add(xmlNode); + } + return elemList; + } + public static String getTextNodeContentOneLine(final Node node) { return getTextNodeContent(node).strip().replaceAll("\r\n\t", ""); } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index a714994..b1c314d 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -14,15 +14,37 @@ server: # Number of selector threads to use. # selectors: - eforms: sdk: # Path to eForms SDK path: eforms-sdk # Comma-separated list of the supported major versions of eForms SDK - versions: 1.8 - + versions: 1.11 + + # Default should be true, it can be set to false for development or experimental purposes. + # If set to false the SDK files must be put manually inside the eforms sdk path + autoDownload: true + +xml: + generation: + + # Generates extra XML attributes for human readability, but makes the XML invalid for eforms. + # It also generates additional files to help debug or understand the XML generation. + # It is recommended to initially set this to true in order to better understand the process. + debug: false + + # Skips items if there is no value set. + # This can help reduce the noise (empty fields ...) in the generated items. + # This is especially useful with an empty form when setting only some values of interest. + # This simplifies the XML files and the conceptual .dot file (assuming debug = true). + # Setting false may break XML validation and unit tests! + skipIfEmpty: true + + # This is related to the use of fields.json "xsdSequenceOrder" to order the XML elements. + # This is mainly used to deactivate the sorting for development purposes. + # It can be used to determine if the sorting is causing an issue or not. + sortXmlElements: true proxy: ### Security: use a command line parameter for security related data diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 5dd45d9..8dff78e 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -32,11 +32,15 @@ - - - + - + + + + + + + diff --git a/src/main/resources/static/css/custom/editor.css b/src/main/resources/static/css/custom/editor.css index f099c77..cddecea 100644 --- a/src/main/resources/static/css/custom/editor.css +++ b/src/main/resources/static/css/custom/editor.css @@ -120,6 +120,24 @@ body { column-gap: 1em; } + +#notice-selector-button { + cursor: pointer; + border: 1px solid var(--eui-primary-100); + color: var(--eui-primary-100); + background-color: var(--eui-white); + padding: .2em 1em .3em 1em !important; + border-radius: .25em; + vertical-align: middle; +} + +#notice-selector { + width: 0px; + height: 0px; + margin: 0px; + padding: 0px; +} + /* Toolbar comboboxes are slightly padded to match the style of other controls in the page */ #toolbar select { padding: 3px 3px; @@ -131,6 +149,7 @@ body { #toolbar label { padding: 0em; margin-right: .25em; + vertical-align: middle; } #loading-indicator { @@ -239,7 +258,20 @@ button { border-radius: var(--small-radius); } -.input-field.container.repeatable { +.repeater { + border: 1px solid var(--eui-primary-100); + border-radius: .5em; + padding: 1em; +} + +.repeater .repeatable.container { + border: dashed 2px var(--eui-primary-25); + border-radius: var(--large-radius); + padding: .25em; + margin: .5em 0em; +} + +.repeatable.input-field.container { border: dashed 2px var(--eui-grey-20); border-radius: var(--large-radius); padding: .25em; @@ -415,6 +447,14 @@ h6 { font-size: 1.0em; } grid-column-end: 3; } +.repeater.container { + grid-column-start: 1; + grid-column-end: 3; + padding: .5em; + border: 1px solid var(--group-border); + border-radius: var(--large-radius); +} + /* * Display groups */ diff --git a/src/main/resources/static/index.html b/src/main/resources/static/index.html index 30420f0..25ab2d4 100644 --- a/src/main/resources/static/index.html +++ b/src/main/resources/static/index.html @@ -11,9 +11,16 @@ + + + @@ -24,9 +31,13 @@

eForms Notice Editor

    +
  • + + +
  • -
    • @@ -36,7 +47,7 @@

    eForms Notice Editor

  • - @@ -52,7 +63,8 @@

    eForms Notice Editor

    - +
    @@ -63,13 +75,16 @@

    eForms Notice Editor

    - - - -