From e4ff871809b348eeb8cdabcfeb77424c72b568e3 Mon Sep 17 00:00:00 2001
From: worksofliam <mrliamallan@live.co.uk>
Date: Mon, 9 Dec 2024 11:49:25 -0500
Subject: [PATCH] Refactor parser inner-functions to remove the use of the
 pieces variable

Signed-off-by: worksofliam <mrliamallan@live.co.uk>
---
 language/parser.ts | 40 ++++++++++++++++++++++------------------
 language/tokens.ts | 14 ++++++++++----
 2 files changed, 32 insertions(+), 22 deletions(-)

diff --git a/language/parser.ts b/language/parser.ts
index ea10181f..613cd862 100644
--- a/language/parser.ts
+++ b/language/parser.ts
@@ -334,10 +334,6 @@ export default class Parser {
       let lineNumber = -1;
       let lineIndex = 0;
 
-      let parts: string[];
-      let partsLower: string[];
-      let pieces: string[];
-
       let isFullyFree = lines[0].toUpperCase().startsWith(`**FREE`);
       let lineIsFree = false;
 
@@ -350,6 +346,16 @@ export default class Parser {
         return directIfScope.length === 0 || directIfScope.every(scope => scope.condition);
       }
 
+      const stripComment = (inputLine: string) => {
+        const comment = inputLine.indexOf(`//`);
+        return (comment >= 0 ? inputLine.substring(0, comment).trimEnd() : inputLine);
+      }
+
+      const getValidStatement = (inputLine: string, withSep?: boolean) => {
+        const sep = inputLine.indexOf(`;`);
+        return (sep >= 0 ? inputLine.substring(0, sep + (withSep ? 1 : 0)) : inputLine);
+      }
+
       const expandDs = async (file: string, ds: Declaration): Promise<void> => {
         const tags = [`LIKEDS`, `LIKEREC`, `EXTNAME`];
         const keywords = ds.keyword;
@@ -475,7 +481,7 @@ export default class Parser {
         } else {
           // Even if the line is useless, we need to capture the characters to be
           // parsed in case it's a statement spread over multiple lines
-          if (currentStmtStart && currentStmtStart.content) {
+          if (!isFullyFree && currentStmtStart && currentStmtStart.content) {
             currentStmtStart.content += ``.padEnd(baseLine.length + EOL.length);
           }
         }
@@ -487,17 +493,15 @@ export default class Parser {
         }
 
         let tokens: Token[] = [];
-        pieces = [];
-        parts = [];
+        let parts: string[];
+        let partsLower: string[];
         
         if (isFullyFree || lineIsFree) {
           // Free format!
           if (line.trim() === ``) continue;
 
-          pieces = line.split(`;`);
-
           const lineIsComment = line.trim().startsWith(`//`);
-          tokens = tokenise(pieces[0], lineNumber, lineIndex);
+          tokens = tokenise(getValidStatement(line), lineNumber, lineIndex);
           partsLower = tokens.filter(piece => piece.value).map(piece => piece.value);
           parts = partsLower.map(piece => piece.toUpperCase());
 
@@ -610,20 +614,19 @@ export default class Parser {
             }
           }
 
-          if (pieces.length > 1 && pieces[1].includes(`//`)) line = pieces[0] + `;`;
 
           if (!currentStmtStart || !currentStmtStart.content) {
             currentStmtStart = {line: lineNumber, index: lineIndex};
           }
 
           if (!lineIsComment) {
-            if (line.endsWith(`;`)) {
+            if (stripComment(line).endsWith(`;`)) {
+
               if (currentStmtStart.content) {
                 // This means the line is just part of the end of the last statement as well.
-                line = currentStmtStart.content + baseLine;
+                line = currentStmtStart.content + getValidStatement(baseLine);
 
-                pieces = line.split(`;`);
-                tokens = tokenise(pieces[0], currentStmtStart.line, currentStmtStart.index);
+                tokens = tokenise(line, currentStmtStart.line, currentStmtStart.index);
                 partsLower = tokens.filter(piece => piece.value).map(piece => piece.value);
                 parts = partsLower.map(piece => piece.toUpperCase());
 
@@ -632,10 +635,11 @@ export default class Parser {
 
             } else if (!line.endsWith(`;`)) {
               currentStmtStart.content = (currentStmtStart.content || ``) + baseLine;
+              
               if (currentStmtStart.content.endsWith(`-`)) 
-                currentStmtStart.content = currentStmtStart.content.substring(0, currentStmtStart.content.length - 1) + LINEEND;
-              else
-                currentStmtStart.content += LINEEND;
+                currentStmtStart.content = currentStmtStart.content.substring(0, currentStmtStart.content.length - 1);
+
+              currentStmtStart.content += LINEEND;
 
               continue;
             }
diff --git a/language/tokens.ts b/language/tokens.ts
index bcb1703c..bb963911 100644
--- a/language/tokens.ts
+++ b/language/tokens.ts
@@ -323,17 +323,23 @@ export function tokenise(statement: string, lineNumber = 0, baseIndex?: number)
       switch (statement[i]) {
       // When it's the string character..
       case stringChar:
+        const possibleEscape = statement[i+1] === stringChar;
         if (state === ReadState.IN_STRING) {
-          currentText += statement[i];
-          result.push({ value: currentText, type: `string`, range: { start: startsAt, end: startsAt + currentText.length, line: lineNumber } });
-          currentText = ``;
+          if (possibleEscape) {
+            currentText += `''`;
+            i += 2;
+          } else {
+            currentText += statement[i];
+            result.push({ value: currentText, type: `string`, range: { start: startsAt, end: startsAt + currentText.length, line: lineNumber } });
+            currentText = ``;
+          }
         } else {
           startsAt = i;
           currentText += statement[i];
         }
 
         // @ts-ignore
-        state = state === ReadState.IN_STRING ? ReadState.NORMAL : ReadState.IN_STRING;
+        state = state === ReadState.IN_STRING && !possibleEscape ? ReadState.NORMAL : ReadState.IN_STRING;
         break;
 
       // When it's any other character...