Fix issue for tabs and long total-free lines

Signed-off-by: worksofliam <[email protected]>

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "tests/sources/BBS400"]
 	path = tests/sources/BBS400
 	url = [email protected]:worksofliam/BBS400.git
+[submodule "tests/sources/noxDB"]
+	path = tests/sources/noxDB
+	url = [email protected]:sitemule/noxDB.git

language/parser.ts

@@ -18,6 +18,16 @@ export type includeFilePromise = (baseFile: string, includeString: string) => Pr
 export type TableDetail = {[name: string]: {fetched: number, fetching?: boolean, recordFormats: Declaration[]}};
 export interface ParseOptions {withIncludes?: boolean, ignoreCache?: boolean, collectReferences?: boolean};
 
+const lineTokens = (input: string, lineNumber: number, lineIndex: number): Token[] => {
+  let tokens = tokenise(input, {
+    baseIndex: lineIndex,
+    lineNumber,
+    ignoreTypes: [`tab`]
+  });
+
+  return tokens;
+}
+
 export default class Parser {
   parsedCache: {[thePath: string]: Cache} = {};
   tables: TableDetail = {};
@@ -487,7 +497,7 @@ export default class Parser {
         }
 
         let line = baseLine;
-        if (line.length > 80) {
+        if (!isFullyFree && line.length > 80) {
           // Remove ending comments
           line = line.substring(0, 80);
         }
@@ -501,7 +511,7 @@ export default class Parser {
           if (line.trim() === ``) continue;
 
           const lineIsComment = line.trim().startsWith(`//`);
-          tokens = tokenise(getValidStatement(line), lineNumber, lineIndex);
+          tokens = lineTokens(getValidStatement(line), lineNumber, lineIndex);
           partsLower = tokens.filter(piece => piece.value).map(piece => piece.value);
           parts = partsLower.map(piece => piece.toUpperCase());
 
@@ -626,7 +636,7 @@ export default class Parser {
                 // This means the line is just part of the end of the last statement as well.
                 line = currentStmtStart.content + getValidStatement(baseLine);
 
-                tokens = tokenise(line, currentStmtStart.line, currentStmtStart.index);
+                tokens = lineTokens(line, currentStmtStart.line, currentStmtStart.index);
                 partsLower = tokens.filter(piece => piece.value).map(piece => piece.value);
                 parts = partsLower.map(piece => piece.toUpperCase());
 
@@ -1257,7 +1267,7 @@ export default class Parser {
             tokens = [cSpec.ind1, cSpec.ind2, cSpec.ind3];
 
             const fromToken = (token?: Token) => {
-              return token ? tokenise(token.value, lineNumber, token.range.start) : [];
+              return token ? lineTokens(token.value, lineNumber, token.range.start) : [];
             };
 
             if (cSpec.opcode && ALLOWS_EXTENDED.includes(cSpec.opcode.value) && !cSpec.factor1 && cSpec.extended) {
@@ -1609,15 +1619,13 @@ export default class Parser {
 
   static getTokens(content: string|string[]|Token[], lineNumber?: number, baseIndex?: number): Token[] {
     if (Array.isArray(content) && typeof content[0] === `string`) {
-      return tokenise(content.join(` `), lineNumber, baseIndex);
+      return lineTokens(content.join(` `), lineNumber, baseIndex);
     } else 
       if (typeof content === `string`) {
-        return tokenise(content, lineNumber, baseIndex);
+        return lineTokens(content, lineNumber, baseIndex);
       } else {
         return content as Token[];
       }
-
-
   }
 
   static expandKeywords(tokens: Token[], isConst = false): Keywords {

language/tokens.ts

@@ -232,7 +232,7 @@ const commonMatchers: Matcher[] = [
   },
 ];
 
-const splitParts = [`%`, `.`, `(`, `)`, `+`, `-`, `*`, `/`, `=`, `:`, `,`, `;`, `\n`, `\r`, ` `];
+const splitParts = [`%`, `.`, `(`, `)`, `+`, `-`, `*`, `/`, `=`, `:`, `,`, `;`, `\n`, `\r`, `\t`, ` `];
 const types = {
   '%': `percent`,
   '.': `dot`,
@@ -248,6 +248,7 @@ const types = {
   ',': `comma`,
   '\n': `newline`,
   '\r': `newliner`,
+  '\t': `tab`,
 };
 
 const stringChar: string = `'`;
@@ -278,10 +279,15 @@ export const ALLOWS_EXTENDED = [
   `XML-SAX`
 ]
 
+export type TokeniseOptions = {lineNumber?: number, baseIndex?: number, ignoreTypes?: string[]};
+
 /**
  * @returns {{value?: string, block?: object[], type: string, position: number}[]}
  */
-export function tokenise(statement: string, lineNumber = 0, baseIndex?: number) {
+export function tokenise(statement: string, options: TokeniseOptions = {}): Token[] {
+  let lineNumber = options.lineNumber || 0;
+  let baseIndex = options.baseIndex || 0;
+
   let commentStart = -1;
   let state: ReadState = ReadState.NORMAL;
 
@@ -351,7 +357,13 @@ export function tokenise(statement: string, lineNumber = 0, baseIndex?: number)
           }
 
           if (statement[i] !== ` `) {
-            result.push({ value: statement[i], type: types[statement[i]], range: { start: i, end: i + statement[i].length, line: lineNumber } });
+            const type = types[statement[i]];
+
+            if (options.ignoreTypes && options.ignoreTypes.includes(type)) {
+              continue;
+            }
+
+            result.push({ value: statement[i], type, range: { start: i, end: i + statement[i].length, line: lineNumber } });
           }
 
           startsAt = i + 1;

tests/suite/partial.test.ts

@@ -33,7 +33,7 @@ test("Parser partial tests", { timeout }, async () => {
         baseContentSplitUpIntoPieces.push(baseContent.substring(i, i + pieceSize));
       }
 
-      console.log(`Testing ${basename} (${i}/${list.length})...`);
+      // console.log(`Testing ${basename} (${i}/${list.length})...`);
 
       let lengths: number[] = [];
       for (let i = 0; i < baseContentSplitUpIntoPieces.length; i++) {
@@ -51,8 +51,8 @@ test("Parser partial tests", { timeout }, async () => {
       const lengthsAverage = lengths.reduce((a, b) => a + b, 0) / lengths.length;
       const total = lengths.reduce((a, b) => a + b, 0);
       const last = lengths[lengths.length - 1];
-      console.log(`\tAverage: ${lengthsAverage}ms, Full: ${last}ms, Total: ${total}`);
-      console.log(``);
+      // console.log(`\tAverage: ${lengthsAverage}ms, Full: ${last}ms, Total: ${total}`);
+      // console.log(``);
     }
   }
 });

tests/suite/references.test.ts

@@ -1410,4 +1410,42 @@ test(`references_21_fixed_exec1`, async () => {
   const tlst = cache.find(`tlst`);
   expect(tlst.references.length).toBe(4);
   expect(tlst.references.every(ref => lines.substring(ref.offset.position, ref.offset.end) === `tlst`)).toBe(true);
+});
+
+test(`references_22_long_lines`, async () => {
+  const lines = [
+    `**free`,
+  `// --------------------------------------------------------------`,
+  `// Next departure from Mols Linien`,
+  `// --------------------------------------------------------------`,
+  `dcl-proc jsonRequest;`,
+  ``,
+  `   dcl-s  pReq   	  	pointer;`,
+  `   dcl-s  pResponse 	pointer;`,
+  `   dcl-s  url  	  	varchar(1024);`,
+  ``,
+  `   // parameters on URL`,
+  `   url = 'https://www.molslinjen.dk/umbraco/api/departure/getnextdepartures?departureRegionId=JYL';`,
+  ``,
+  `   // Note: No payload in the request. use *null - here we pass a null pointer  `,
+  `   // Note: No options in the request. use *null - here we pass the *null literal value`,
+  `   `,
+  `   // Do the http request to get next depature`,
+  `   // Use YUM to install curl, which is the tool used by httpRequest`,
+  `   pResponse = json_httpRequest (url: pReq:*null:'JSON');`,
+  ``,
+  `   json_WriteJsonStmf(pResponse:'/prj/noxdb/testout/httpdump.json':1208:*OFF);`,
+  ``,
+  `   json_delete(pReq);`,
+  `   json_delete(pResponse);`,
+  ``,
+  `end-proc;`,
+  ].join(`\n`);
+
+  const cache = await parser.getDocs(uri, lines, { ignoreCache: true, withIncludes: true, collectReferences: true });
+
+  const jsonRequest = cache.find(`jsonRequest`);
+  const pReq = jsonRequest.scope.find(`pReq`);
+  expect(pReq.references.length).toBe(3);
+  expect(pReq.references.every(ref => lines.substring(ref.offset.position, ref.offset.end) === `pReq`)).toBe(true);
 });

tests/suite/sources.test.ts

@@ -91,7 +91,7 @@ test("Generic reference tests", { timeout }, async () => {
         fail(`Found ${errorCount} errors in ${basename}`);
       }
 
-      console.log(`Parsed ${basename} in ${pe - ps}ms. Validated in ${se-ss} (${i+1}/${list.length}). Found ${referencesCollected} references.`);
+      // console.log(`Parsed ${basename} in ${pe - ps}ms. Validated in ${se-ss} (${i+1}/${list.length}). Found ${referencesCollected} references.`);
     }
   }
 });