Merge pull request #131 from motdotla/better-expansion

12.0.0 better expansion to match dotenvx's
motdotla · Nov 16, 2024 · 07a3a44 · 07a3a44
2 parents 3213210 + a3b31a8
commit 07a3a44
Show file tree

Hide file tree

Showing 5 changed files with 109 additions and 78 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,7 +2,19 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
-## [Unreleased](https://github.com/motdotla/dotenv-expand/compare/v11.0.7...master)
+## [Unreleased](https://github.com/motdotla/dotenv-expand/compare/v12.0.0...master)
+
+## [12.0.0](https://github.com/motdotla/dotenv-expand/compare/v11.0.7...v12.0.0) (2024-11-16)
+
+### Added
+
+* 🎉 support alternate value expansion (see [usage](https://dotenvx.com/docs/env-file#interpolation)) ([#131](https://github.com/motdotla/dotenv-expand/pull/131))
+
+### Changed
+
+* 🎉 Expansion logic rewritten to match [dotenvx's](https://github.com/dotenvx/dotenvx). (*note: I recommend dotenvx over dotenv-expand when you are ready. I'm putting all my effort there for a unified standard .env implementation that works everywhere and matches bash, docker-compose, and more. In some cases it slightly improves on them. This leads to more reliability for your secrets and config.) ([#131](https://github.com/motdotla/dotenv-expand/pull/131))
+* ⚠️ BREAKING: do NOT expand in reverse order. Instead, order your .env file keys from first to last as they depend on each other for expansion - principle of least surprise. ([#131](https://github.com/motdotla/dotenv-expand/pull/131))
+* ⚠️ BREAKING: do NOT attempt expansion of process.env. This has always been dangerous (unexpected side effects) and is now removed. process.env should not hold values you want to expand. Put expansion logic in your .env file. If you need this ability, use [dotenvx](https://github.com/dotenvx/dotenvx) by shipping an encrypted .env file with your code - allowing safe expansion at runtime. ([#131](https://github.com/motdotla/dotenv-expand/pull/131))
 
 ## [11.0.7](https://github.com/motdotla/dotenv-expand/compare/v11.0.6...v11.0.7) (2024-11-13)
 

diff --git a/README.md b/README.md
@@ -1,5 +1,5 @@
 <div align="center">
-🎉 announcing <a href="https://github.com/dotenvx/dotenvx">dotenvx</a>. <em><b>better expansion</b>, run anywhere, multi-environment, encrypted envs</em>.
+🎉 announcing <a href="https://github.com/dotenvx/dotenvx">dotenvx</a>. <em><b>expansion AND command substitution</b>, multi-environment, encrypted envs, and more</em>.
 </div>
 
 &nbsp;
@@ -166,28 +166,13 @@ console.log(process.env.HELLO) // undefined
 
 ### What rules does the expansion engine follow?
 
-The expansion engine roughly has the following rules:
-
-* `$KEY` will expand any env with the name `KEY`
-* `${KEY}` will expand any env with the name `KEY` 
-* `\$KEY` will escape the `$KEY` rather than expand
-* `${KEY:-default}` will first attempt to expand any env with the name `KEY`. If not one, then it will return `default`
-* `${KEY-default}` will first attempt to expand any env with the name `KEY`. If not one, then it will return `default`
-
-You can see a full list of rules [here](https://dotenvx.com/docs/env-file#interpolation).
+See a full list of rules [here](https://dotenvx.com/docs/env-file#interpolation).
 
 ### How can I avoid expanding pre-existing envs (already in my `process.env`, for example `pas$word`)?
 
-Modify your `dotenv.config` to write to an empty object and pass that to `dotenvExpand.processEnv`.
-
-```js
-const dotenv = require('dotenv')
-const dotenvExpand = require('dotenv-expand')
-
-const myEnv = dotenv.config({ processEnv: {} }) // prevent writing to `process.env`
+As of `v12.0.0` dotenv-expand no longer expands `process.env`.
 
-dotenvExpand.expand(myEnv)
-```
+If you need this ability, use [dotenvx](https://github.com/dotenvx/dotenvx) by shipping an encrypted .env file with your code - allowing safe expansion at runtime.
 
 ## Contributing Guide
 

diff --git a/lib/main.js b/lib/main.js
@@ -1,79 +1,91 @@
 'use strict'
 
-// * /
-// *   (\\)?            # is it escaped with a backslash?
-// *   (\$)             # literal $
-// *   (?!\()           # shouldnt be followed by parenthesis
-// *   (\{?)            # first brace wrap opening
-// *   ([\w.]+)         # key
-// *   (?::-((?:\$\{(?:\$\{(?:\$\{[^}]*\}|[^}])*}|[^}])*}|[^}])+))? # optional default nested 3 times
-// *   (\}?)            # last brace warp closing
-// * /xi
-
-const DOTENV_SUBSTITUTION_REGEX = /(\\)?(\$)(?!\()(\{?)([\w.]+)(?::?-((?:\$\{(?:\$\{(?:\$\{[^}]*\}|[^}])*}|[^}])*}|[^}])+))?(\}?)/gi
-
 function _resolveEscapeSequences (value) {
   return value.replace(/\\\$/g, '$')
 }
 
-function interpolate (value, processEnv, parsed) {
-  return value.replace(DOTENV_SUBSTITUTION_REGEX, (match, escaped, dollarSign, openBrace, key, defaultValue, closeBrace) => {
-    if (escaped === '\\') {
-      return match.slice(1)
-    } else {
-      if (processEnv[key]) {
-        if (processEnv[key] === parsed[key]) {
-          return processEnv[key]
-        } else {
-          // scenario: PASSWORD_EXPAND_NESTED=${PASSWORD_EXPAND}
-          return interpolate(processEnv[key], processEnv, parsed)
-        }
-      }
+function expandValue (value, processEnv, runningParsed) {
+  const env = { ...runningParsed, ...processEnv } // process.env wins
 
-      if (parsed[key]) {
-        // avoid recursion from EXPAND_SELF=$EXPAND_SELF
-        if (parsed[key] !== value) {
-          return interpolate(parsed[key], processEnv, parsed)
-        }
-      }
+  const regex = /(?<!\\)\${([^{}]+)}|(?<!\\)\$([A-Za-z_][A-Za-z0-9_]*)/g
+
+  let result = value
+  let match
+  const seen = new Set() // self-referential checker
+
+  while ((match = regex.exec(result)) !== null) {
+    seen.add(result)
+
+    const [template, bracedExpression, unbracedExpression] = match
+    const expression = bracedExpression || unbracedExpression
+
+    // match the operators `:+`, `+`, `:-`, and `-`
+    const opRegex = /(:\+|\+|:-|-)/
+    // find first match
+    const opMatch = expression.match(opRegex)
+    const splitter = opMatch ? opMatch[0] : null
+
+    const r = expression.split(splitter)
+
+    let defaultValue
+    let value
+
+    const key = r.shift()
+
+    if ([':+', '+'].includes(splitter)) {
+      defaultValue = env[key] ? r.join(splitter) : ''
+      value = null
+    } else {
+      defaultValue = r.join(splitter)
+      value = env[key]
+    }
 
-      if (defaultValue) {
-        if (defaultValue.startsWith('$')) {
-          return interpolate(defaultValue, processEnv, parsed)
-        } else {
-          return defaultValue
-        }
+    if (value) {
+      // self-referential check
+      if (seen.has(value)) {
+        result = result.replace(template, defaultValue)
+      } else {
+        result = result.replace(template, value)
       }
+    } else {
+      result = result.replace(template, defaultValue)
+    }
 
-      return ''
+    // if the result equaled what was in process.env and runningParsed then stop expanding
+    if (result === processEnv[key] && result === runningParsed[key]) {
+      break
     }
-  })
+
+    regex.lastIndex = 0 // reset regex search position to re-evaluate after each replacement
+  }
+
+  return result
 }
 
 function expand (options) {
+  // for use with progressive expansion
+  const runningParsed = {}
+
   let processEnv = process.env
   if (options && options.processEnv != null) {
     processEnv = options.processEnv
   }
 
+  // dotenv.config() ran before this so the assumption is process.env has already been set
   for (const key in options.parsed) {
     let value = options.parsed[key]
 
-    const inProcessEnv = Object.prototype.hasOwnProperty.call(processEnv, key)
-    if (inProcessEnv) {
-      if (processEnv[key] === options.parsed[key]) {
-        // assume was set to processEnv from the .env file if the values match and therefore interpolate
-        value = interpolate(value, processEnv, options.parsed)
-      } else {
-        // do not interpolate - assume processEnv had the intended value even if containing a $.
-        value = processEnv[key]
-      }
+    // short-circuit scenario: process.env was already set prior to the file value
+    if (processEnv[key] && processEnv[key] !== value) {
+      value = processEnv[key]
     } else {
-      // not inProcessEnv so assume interpolation for this .env key
-      value = interpolate(value, processEnv, options.parsed)
+      value = expandValue(value, processEnv, runningParsed)
     }
 
     options.parsed[key] = _resolveEscapeSequences(value)
+
+    // for use with progressive expansion
+    runningParsed[key] = _resolveEscapeSequences(value)
   }
 
   for (const processKey in options.parsed) {

diff --git a/tests/.env.test b/tests/.env.test
@@ -79,3 +79,6 @@ PASSWORD_EXPAND=${PASSWORD}
 PASSWORD_EXPAND_SIMPLE=$PASSWORD
 PASSWORD_EXPAND_NESTED=${PASSWORD_EXPAND}
 PASSWORD_EXPAND_NESTED_NESTED=${PASSWORD_EXPAND_NESTED}
+
+USE_IF_SET=true
+ALTERNATE=${USE_IF_SET:+alternate}
diff --git a/tests/main.js b/tests/main.js
@@ -404,8 +404,8 @@ t.test('should expand with default value correctly', ct => {
 
   ct.equal(parsed.UNDEFINED_EXPAND_DEFAULT_SPECIAL_CHARACTERS, '/default/path:with/colon')
   ct.equal(parsed.UNDEFINED_EXPAND_DEFAULT_SPECIAL_CHARACTERS2, '/default/path:with/colon')
-  ct.equal(parsed.NO_CURLY_BRACES_UNDEFINED_EXPAND_DEFAULT_SPECIAL_CHARACTERS, '/default/path:with/colon')
-  ct.equal(parsed.NO_CURLY_BRACES_UNDEFINED_EXPAND_DEFAULT_SPECIAL_CHARACTERS2, '/default/path:with/colon')
+  ct.equal(parsed.NO_CURLY_BRACES_UNDEFINED_EXPAND_DEFAULT_SPECIAL_CHARACTERS, ':-/default/path:with/colon')
+  ct.equal(parsed.NO_CURLY_BRACES_UNDEFINED_EXPAND_DEFAULT_SPECIAL_CHARACTERS2, '-/default/path:with/colon')
 
   ct.end()
 })
@@ -454,7 +454,7 @@ t.test('handles two dollar signs', ct => {
   const dotenv = require('dotenv').config({ path: 'tests/.env.test', processEnv: {} })
   const parsed = dotenvExpand.expand(dotenv).parsed
 
-  ct.equal(parsed.TWO_DOLLAR_SIGNS, 'abcd$')
+  ct.equal(parsed.TWO_DOLLAR_SIGNS, 'abcd$$1234')
 
   ct.end()
 })
@@ -535,7 +535,7 @@ t.test('expands recursively', ct => {
   ct.end()
 })
 
-t.test('expands recursively reverse order', ct => {
+t.test('CANNOT expand recursively reverse order (ORDER YOUR .env file for least surprise)', ct => {
   const dotenv = {
     parsed: {
       BACKEND_API_HEALTH_CHECK_URL: '${MOCK_SERVER_HOST}/ci-health-check',
@@ -546,8 +546,8 @@ t.test('expands recursively reverse order', ct => {
   const parsed = dotenvExpand.expand(dotenv).parsed
 
   ct.equal(parsed.MOCK_SERVER_PORT, '8090')
-  ct.equal(parsed.MOCK_SERVER_HOST, 'http://localhost:8090')
-  ct.equal(parsed.BACKEND_API_HEALTH_CHECK_URL, 'http://localhost:8090/ci-health-check')
+  ct.equal(parsed.MOCK_SERVER_HOST, 'http://localhost:')
+  ct.equal(parsed.BACKEND_API_HEALTH_CHECK_URL, '/ci-health-check')
 
   ct.end()
 })
@@ -571,11 +571,30 @@ t.test('expands recursively but is smart enough to not attempt expansion of a pr
   const dotenv = require('dotenv').config({ path: 'tests/.env.test' })
   dotenvExpand.expand(dotenv)
 
+  ct.equal(process.env.PASSWORD, 'pas$word')
   ct.equal(process.env.PASSWORD_EXPAND, 'pas$word')
   ct.equal(process.env.PASSWORD_EXPAND_SIMPLE, 'pas$word')
-  ct.equal(process.env.PASSWORD, 'pas$word')
-  ct.equal(process.env.PASSWORD_EXPAND_NESTED, 'pas$word')
   ct.equal(process.env.PASSWORD_EXPAND_NESTED, 'pas$word')
+  ct.equal(process.env.PASSWORD_EXPAND_NESTED_NESTED, 'pas$word')
+
+  ct.end()
+})
+
+t.test('expands alternate logic', ct => {
+  const dotenv = require('dotenv').config({ path: 'tests/.env.test' })
+  dotenvExpand.expand(dotenv)
+
+  ct.equal(process.env.ALTERNATE, 'alternate')
+
+  ct.end()
+})
+
+t.test('expands alternate logic when not set', ct => {
+  process.env.USE_IF_SET = ''
+  const dotenv = require('dotenv').config({ path: 'tests/.env.test' })
+  dotenvExpand.expand(dotenv)
+
+  ct.equal(process.env.ALTERNATE, '')
 
   ct.end()
 })