6345 - Uplift to enketo-core version 5.18.1

Gruntfile.js

@@ -541,11 +541,12 @@ module.exports = function(grunt) {
             'patch webapp/node_modules/moment/locale/hi.js < webapp/patches/moment-hindi-use-euro-numerals.patch',
 
             // patch enketo to always mark the /inputs group as relevant
-            'patch webapp/node_modules/enketo-core/src/js/Form.js < webapp/patches/enketo-inputs-always-relevant.patch',
+            'patch webapp/node_modules/enketo-core/src/js/form.js < webapp/patches/enketo-inputs-always-relevant_form.patch',
+            'patch webapp/node_modules/enketo-core/src/js/relevant.js < webapp/patches/enketo-inputs-always-relevant_relevant.patch',
 
-            // patch enketo so forms with no active pages are considered valid
-            // https://github.com/medic/medic/issues/5484
-            'patch webapp/node_modules/enketo-core/src/js/page.js < webapp/patches/enketo-handle-no-active-pages.patch',
+            // patch enketo to fix repeat name collision bug - this should be removed when upgrading to a new version of enketo-core
+            // https://github.com/enketo/enketo-core/issues/815
+            'patch webapp/node_modules/enketo-core/src/js/calculate.js < webapp/patches/enketo-repeat-name-collision.patch',
 
             // patch messageformat to add a default plural function for languages not yet supported by make-plural #5705
             'patch webapp/node_modules/messageformat/lib/plurals.js < webapp/patches/messageformat-default-plurals.patch',

api/package.json

@@ -21,7 +21,7 @@
     "body-parser": "^1.19.2",
     "buffer-shims": "^1.0.0",
     "compression": "^1.7.4",
-    "enketo-xslt": "^1.15.2",
+    "enketo-transformer": "^2.0.0",
     "express": "^4.17.1",
     "google-libphonenumber": "^3.2.30",
     "gsm": "^0.1.4",
@@ -33,6 +33,7 @@
     "morgan": "^1.10.0",
     "mustache": "^4.2.0",
     "node-cache": "^5.1.2",
+    "node-html-parser": "^3.0.4",
     "object-path": "^0.11.8",
     "openrosa-formlist": "https://github.com/medic/openrosa-formlist#sax",
     "pass-stream": "^1.0.0",

api/src/services/generate-xform.js

@@ -4,18 +4,19 @@
  */
 const childProcess = require('child_process');
 const path = require('path');
+const htmlParser = require('node-html-parser');
 const logger = require('../logger');
 const db = require('../db');
 const formsService = require('./forms');
+const markdown = require('enketo-transformer/src/markdown');
 
-const FORM_ROOT_OPEN = '<root xmlns:xf="http://www.w3.org/2002/xforms" xmlns:orx="http://openrosa.org/xforms" xmlns:enk="http://enketo.org/xforms" xmlns:kb="http://kobotoolbox.org/xforms" xmlns:esri="http://esri.com/xforms" xmlns:oc="http://openclinica.org/xforms" xmlns:h="http://www.w3.org/1999/xhtml" xmlns:ev="http://www.w3.org/2001/xml-events" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:jr="http://openrosa.org/javarosa">';
 const MODEL_ROOT_OPEN = '<root xmlns="http://www.w3.org/2002/xforms" xmlns:xf="http://www.w3.org/2002/xforms" xmlns:h="http://www.w3.org/1999/xhtml" xmlns:ev="http://www.w3.org/2001/xml-events" xmlns:xsd="http://www.w3.org/2001/XMLSchema">';
 const ROOT_CLOSE = '</root>';
 const JAVAROSA_SRC = / src="jr:\/\//gi;
 const MEDIA_SRC_ATTR = ' data-media-src="';
 
 const FORM_STYLESHEET = path.join(__dirname, '../xsl/openrosa2html5form.xsl');
-const MODEL_STYLESHEET = path.join(__dirname, '../../node_modules/enketo-xslt/xsl/openrosa2xmlmodel.xsl');
+const MODEL_STYLESHEET = path.join(__dirname, '../../node_modules/enketo-transformer/src/xsl/openrosa2xmlmodel.xsl');
 const XSLTPROC_CMD = 'xsltproc';
 
 const processErrorHandler = (xsltproc, err, reject) => {
@@ -70,33 +71,93 @@ const transform = (formXml, stylesheet) => {
   });
 };
 
-const removeLast = (haystack, needle) => {
-  const index = haystack.lastIndexOf(needle);
-  if (index === -1) {
-    return haystack;
-  }
-  return haystack.slice(0, index) + haystack.slice(index + needle.length);
+const convertDynamicUrls = (original) => original.replace(
+  /<a[^>]+href="([^"]*---output[^"]*)"[^>]*>(.*?)<\/a>/gm,
+  '<a href="#" target="_blank" rel="noopener" class="dynamic-url">' +
+  '$2<span class="url hidden">$1</span>' +
+  '</a>');
+
+const convertEmbeddedHtml = (original) => original
+  .replace(/&lt;\s*(\/)?\s*([\s\S]*?)\s*&gt;/gm, '<$1$2>')
+  .replace(/&quot;/g, '"')
+  .replace(/&#039;/g, '\'')
+  .replace(/&amp;/g, '&');
+
+const replaceNode = (currentNode, newNode) => {
+  const { parentNode } = currentNode;
+  const idx = parentNode.childNodes.findIndex((child) => child === currentNode);
+  parentNode.childNodes = [
+    ...parentNode.childNodes.slice(0, idx),
+    newNode,
+    ...parentNode.childNodes.slice(idx + 1),
+  ];
 };
 
-const removeRootNode = (string, node) => {
-  return removeLast(string.replace(node, ''), ROOT_CLOSE);
+// Based on enketo/enketo-transformer
+// https://github.com/enketo/enketo-transformer/blob/377caf14153586b040367f8c2de53c9d794c19d4/src/transformer.js#L430
+const replaceAllMarkdown = (formString) => {
+  const replacements = {};
+  const form = htmlParser.parse(formString).querySelector('form');
+
+  // First turn all outputs into text so *<span class="or-output></span>* can be detected
+  form.querySelectorAll('span.or-output').forEach((el, index) => {
+    const key = `---output-${index}`;
+    const textNode = el.childNodes[0];
+    replacements[key] = el.toString();
+    textNode.textContent = key;
+    replaceNode(el, textNode);
+    // Note that we end up in a situation where we likely have sibling text nodes...
+  });
+
+  // Now render markdown
+  const questions = form.querySelectorAll('span.question-label');
+  const hints = form.querySelectorAll('span.or-hint');
+  questions.concat(hints).forEach((el, index) => {
+    const original = el.innerHTML;
+    let rendered = markdown.toHtml(original);
+    rendered = convertDynamicUrls(rendered);
+    rendered = convertEmbeddedHtml(rendered);
+
+    if (original !== rendered) {
+      const key = `$$$${index}`;
+      replacements[key] = rendered;
+      el.innerHTML = key;
+    }
+  });
+
+  let result = form.toString();
+
+  // Now replace the placeholders with the rendered HTML
+  // in reverse order so outputs are done last
+  Object.keys(replacements).reverse().forEach(key => {
+    const replacement = replacements[key];
+    if (replacement) {
+      result = result.replace(key, replacement);
+    }
+  });
+
+  return result;
 };
 
 const generateForm = formXml => {
   return transform(formXml, FORM_STYLESHEET).then(form => {
+    form = replaceAllMarkdown(form);
     // rename the media src attributes so the browser doesn't try and
     // request them, instead leaving it to custom code in the Enketo
     // service to load them asynchronously
-    form = form.replace(JAVAROSA_SRC, MEDIA_SRC_ATTR);
-    // remove the root node leaving just the HTML to be rendered
-    return removeRootNode(form, FORM_ROOT_OPEN);
+    return form.replace(JAVAROSA_SRC, MEDIA_SRC_ATTR);
   });
 };
 
 const generateModel = formXml => {
   return transform(formXml, MODEL_STYLESHEET).then(model => {
     // remove the root node leaving just the model
-    return removeRootNode(model, MODEL_ROOT_OPEN);
+    model = model.replace(MODEL_ROOT_OPEN, '');
+    const index = model.lastIndexOf(ROOT_CLOSE);
+    if (index === -1) {
+      return model;
+    }
+    return model.slice(0, index) + model.slice(index + ROOT_CLOSE.length);
   });
 };
 

api/src/xsl/openrosa2html5form.xsl

@@ -2,24 +2,186 @@
 <!--
 This stylesheet extends the default one to allow for additional input types.
 -->
-<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                xmlns:xf="http://www.w3.org/2002/xforms"
+                xmlns:enk="http://enketo.org/xforms"
+                xmlns:jr="http://openrosa.org/javarosa"
+                xmlns:orx="http://openrosa.org/xforms">
 
-  <xsl:import href="../../node_modules/enketo-xslt/xsl/openrosa2html5form.xsl"/>
-
+  <xsl:import href="../../node_modules/enketo-transformer/src/xsl/openrosa2html5form.xsl"/>
+
+  <!-- Overwrite binding-attributes declaration from openrosa2html5form.xsl to include custom code -->
+  <!-- Prevent notes from ever being required -->
+  <xsl:template name="binding-attributes">
+    <xsl:param name="binding"/>
+    <xsl:param name="nodeset"/>
+    <xsl:param name="type"/>
+    <xsl:variable name="xml-type">
+      <xsl:call-template name="xml_type">
+        <xsl:with-param name="nodeset" select="$nodeset"/>
+        <!--<xsl:with-param name="binding" select="$binding"/>-->
+      </xsl:call-template>
+    </xsl:variable>
+    <xsl:variable name="html-input-type">
+      <xsl:call-template name="html_type">
+        <xsl:with-param name="xml_type" select="$xml-type" />
+      </xsl:call-template>
+    </xsl:variable>
+    <xsl:choose>
+      <xsl:when test="$type = 'select_multiple'">
+        <xsl:attribute name="multiple">multiple</xsl:attribute>
+      </xsl:when>
+      <xsl:when test="$type = 'select_one'"></xsl:when>
+      <xsl:when test="$type = 'textarea'"></xsl:when>
+      <xsl:when test="$type = 'rank'">
+        <xsl:attribute name="type">rank</xsl:attribute>
+      </xsl:when>
+      <xsl:otherwise>
+        <xsl:attribute name="type">
+          <xsl:value-of select="$html-input-type"/>
+        </xsl:attribute>
+      </xsl:otherwise>
+    </xsl:choose>
+    <xsl:attribute name="name">
+      <xsl:value-of select="normalize-space($nodeset)" />
+    </xsl:attribute>
+    <xsl:if test="$html-input-type = 'radio'">
+      <xsl:attribute name="data-name">
+        <xsl:value-of select="normalize-space($nodeset)" />
+      </xsl:attribute>
+    </xsl:if>
+    <xsl:if test="local-name() = 'item'">
+      <xsl:attribute name="value">
+        <xsl:value-of select="./xf:value"/>
+      </xsl:attribute>
+    </xsl:if>
+    <!-- Medic specific section start -->
+    <!-- Do not copy the required attribute for notes -->
+    <xsl:if test="(string-length($binding/@required) &gt; 0) and not($binding/@required = 'false()') and not(local-name() = 'bind') and not($binding/@type='string' and $binding/@readonly='true()' and not(string-length($binding/@calculate) &gt; 0))">
+    <!-- Medic specific section end -->
+      <xsl:attribute name="data-required">
+        <xsl:value-of select="$binding/@required" />
+      </xsl:attribute>
+    </xsl:if>
+    <xsl:if test="$binding/@constraint">
+      <xsl:attribute name="data-constraint">
+        <xsl:value-of select="$binding/@constraint" />
+      </xsl:attribute>
+    </xsl:if>
+    <xsl:if test="$binding/@relevant">
+      <xsl:attribute name="data-relevant">
+        <xsl:value-of select="$binding/@relevant"/>
+      </xsl:attribute>
+    </xsl:if>
+    <xsl:if test="$binding/@calculate">
+      <xsl:attribute name="data-calculate">
+        <xsl:value-of select="$binding/@calculate" />
+      </xsl:attribute>
+    </xsl:if>
+    <xsl:if test="$binding/@jr:preload">
+      <xsl:attribute name="data-preload">
+        <xsl:value-of select="$binding/@jr:preload"/>
+      </xsl:attribute>
+      <xsl:attribute name="data-preload-params">
+        <xsl:value-of select="$binding/@jr:preloadParams"/>
+      </xsl:attribute>
+    </xsl:if>
+    <xsl:if test="$binding/@enk:for">
+      <xsl:attribute name="data-for">
+        <xsl:value-of select="normalize-space($binding/@enk:for)" />
+      </xsl:attribute>
+    </xsl:if>
+    <xsl:if test="$openclinica = 1">
+      <xsl:for-each select="$binding/@*[starts-with(name(), 'oc:') and not(substring-before(name(), 'Msg'))]" >
+        <xsl:attribute name="{concat('data-oc-', local-name(.))}">
+          <xsl:value-of select="normalize-space(.)" />
+        </xsl:attribute>
+      </xsl:for-each>
+    </xsl:if>
+    <xsl:if test="$binding/@orx:max-pixels">
+      <xsl:attribute name="data-max-pixels">
+        <xsl:value-of select="normalize-space($binding/@orx:max-pixels)" />
+      </xsl:attribute>
+    </xsl:if>
+    <xsl:attribute name="data-type-xml">
+      <xsl:value-of select="$xml-type" />
+    </xsl:attribute>
+    <xsl:if test="$xml-type = 'decimal'">
+      <xsl:attribute name="step">any</xsl:attribute>
+    </xsl:if>
+    <xsl:if test="$binding/@readonly = 'true()' and not($html-input-type = 'hidden')" >
+      <!--
+          This also adds a readonly attribute to <select> which is not valid HTML.
+          We could add some logic to avoid that (the <option>s already get the disabled attribute),
+          but it's an extra line of defence and doesn't really hurt. The input change handler in
+          Enketo Core ignores changes on a <select readonly>.
+      -->
+      <xsl:attribute name="readonly">readonly</xsl:attribute>
+    </xsl:if>
+    <xsl:if test="local-name() = 'range'">
+      <!-- note that due to the unhelpful default value behavior of input type=range in HTML, we use type=number -->
+      <xsl:if test="@start">
+        <xsl:attribute name="min">
+          <xsl:value-of select="@start" />
+        </xsl:attribute>
+      </xsl:if>
+      <xsl:if test="@end">
+        <xsl:attribute name="max">
+          <xsl:value-of select="@end" />
+        </xsl:attribute>
+      </xsl:if>
+      <xsl:if test="@step">
+        <xsl:attribute name="step">
+          <xsl:value-of select="@step" />
+        </xsl:attribute>
+      </xsl:if>
+    </xsl:if>
+    <xsl:if test="$html-input-type = 'file'">
+      <xsl:attribute name="accept">
+        <xsl:choose>
+          <xsl:when test="@accept">
+            <xsl:value-of select="@accept" />
+          </xsl:when>
+          <xsl:when test="@mediatype">
+            <xsl:value-of select="@mediatype" />
+          </xsl:when>
+        </xsl:choose>
+      </xsl:attribute>
+      <!-- Note, this test captures new, new-front, new-rear -->
+      <xsl:if test="contains(@appearance, 'new')">
+        <xsl:attribute name="capture">
+          <xsl:choose>
+            <xsl:when test="contains(@appearance, 'new-front')">
+              <xsl:value-of select="'user'"/>
+            </xsl:when>
+            <xsl:when test="contains(@appearance, 'new-rear')">
+              <xsl:value-of select="'environment'"/>
+            </xsl:when>
+            <!-- else (if appearance="new"), the capture attribute remains empty, by design -->
+          </xsl:choose>
+        </xsl:attribute>
+      </xsl:if>
+    </xsl:if>
+  </xsl:template>
+
+  <!-- Overwrite html_type declaration from openrosa2html5form.xsl to include custom code -->
+  <!-- Allow custom Medic types -->
   <xsl:template name="html_type">
     <xsl:param name="xml_type" />
     <xsl:choose>
       <xsl:when test="local-name(..) = 'select1' or $xml_type='select1' or local-name(.) = 'trigger'">radio</xsl:when>
       <xsl:when test="local-name(..) = 'select' or $xml_type='select'">checkbox</xsl:when>
       <xsl:when test="local-name() = 'bind'">hidden</xsl:when>
-      <xsl:when test="$xml_type = 'dateTime'">datetime</xsl:when>
+      <xsl:when test="local-name() = 'range'">number</xsl:when>
+      <xsl:when test="$xml_type = 'dateTime'">datetime-local</xsl:when>
       <xsl:when test="$xml_type = 'date'">date</xsl:when>
       <!-- note, it may not actually be possible to support 'file' with offline storage -->
       <xsl:when test="$xml_type = 'binary'">file</xsl:when>
       <xsl:when test="$xml_type = 'time'">time</xsl:when>
+      <xsl:when test="$xml_type = 'rank'">text</xsl:when>
       <xsl:when
-        test="$xml_type = 'decimal' or $xml_type = 'float' or $xml_type = 'double' or $xml_type = 'int' or $xml_type = 'integer'"
-        >number</xsl:when>
+              test="$xml_type = 'decimal' or $xml_type = 'float' or $xml_type = 'double' or $xml_type = 'int' or $xml_type = 'integer'"
+      >number</xsl:when>
       <xsl:when test="$xml_type = 'string' and contains(./@appearance, 'numbers')">tel</xsl:when>
       <xsl:when test="$xml_type = 'string'">text</xsl:when>
       <xsl:when test="$xml_type = 'barcode' or $xml_type = 'geopoint' or $xml_type = 'geotrace' or $xml_type = 'geoshape'" >