Enhance parser for reference collection and testing improvements

.github/workflows/test.yaml

@@ -2,6 +2,7 @@ on:
   pull_request:
     paths:
       - 'language/**'
+      - 'tests/**'
 
 jobs:
   release:
@@ -11,6 +12,7 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 1
+          submodules: true
       - run: npm install
       - run: node ./node_modules/eslint/bin/eslint src/** --no-error-on-unmatched-pattern
       - run: npm run test

.gitmodules

@@ -0,0 +1,24 @@
+[submodule "tests/sources/bob-recursive-example"]
+	path = tests/sources/bob-recursive-example
+	url = [email protected]:IBM/bob-recursive-example.git
+[submodule "tests/sources/BBS400"]
+	path = tests/sources/BBS400
+	url = [email protected]:worksofliam/BBS400.git
+[submodule "tests/sources/noxDB"]
+	path = tests/sources/noxDB
+	url = [email protected]:sitemule/noxDB.git
+[submodule "tests/sources/xmlservice"]
+	path = tests/sources/xmlservice
+	url = [email protected]:IBM/xmlservice.git
+[submodule "tests/sources/I_builder"]
+	path = tests/sources/I_builder
+	url = [email protected]:ibmiiste/I_builder.git
+[submodule "tests/sources/ibmi-company_system"]
+	path = tests/sources/ibmi-company_system
+	url = [email protected]:IBM/ibmi-company_system.git
+[submodule "tests/sources/httpapi"]
+	path = tests/sources/httpapi
+	url = [email protected]:ScottKlement/httpapi.git
+[submodule "tests/sources/rpgle-repl"]
+	path = tests/sources/rpgle-repl
+	url = [email protected]:tom-writes-code/rpgle-repl.git

extension/server/src/connection.ts

@@ -63,15 +63,23 @@ export async function memberResolve(baseUri: string, member: string, file: strin
 
 	if (resolvedMembers[baseUri] && resolvedMembers[baseUri][fileKey]) return resolvedMembers[baseUri][fileKey];
 
-	const resolvedMember = await queue.add(() => {return connection.sendRequest("memberResolve", [member, file])}) as IBMiMember|undefined;
-	// const resolvedMember = await connection.sendRequest("memberResolve", [member, file]) as IBMiMember|undefined;
-
-	if (resolvedMember) {
-		if (!resolvedMembers[baseUri]) resolvedMembers[baseUri] = {};
-		resolvedMembers[baseUri][fileKey] = resolvedMember;
+	try {
+		const resolvedMember = await queue.add(() => {return connection.sendRequest("memberResolve", [member, file])}) as IBMiMember|undefined;
+		// const resolvedMember = await connection.sendRequest("memberResolve", [member, file]) as IBMiMember|undefined;
+
+		if (resolvedMember) {
+			if (!resolvedMembers[baseUri]) resolvedMembers[baseUri] = {};
+			resolvedMembers[baseUri][fileKey] = resolvedMember;
+		}
+
+		return resolvedMember;
+	} catch (e) {
+		console.log(`Member resolve failed.`);
+		console.log(JSON.stringify({baseUri, member, file}));
+		console.log(e);
 	}
 
-	return resolvedMember;
+	return undefined;
 }
 
 export async function streamfileResolve(baseUri: string, base: string[]): Promise<string|undefined> {
@@ -82,15 +90,23 @@ export async function streamfileResolve(baseUri: string, base: string[]): Promis
 
 	const paths = (workspace ? includePath[workspace.uri] : []) || [];
 
-	const resolvedPath = await queue.add(() => {return connection.sendRequest("streamfileResolve", [base, paths])}) as string|undefined;
-	//  const resolvedPath = await connection.sendRequest("streamfileResolve", [base, paths]) as string|undefined;
+	try {
+		const resolvedPath = await queue.add(() => {return connection.sendRequest("streamfileResolve", [base, paths])}) as string|undefined;
+		//  const resolvedPath = await connection.sendRequest("streamfileResolve", [base, paths]) as string|undefined;
+
+		if (resolvedPath) {
+			if (!resolvedStreamfiles[baseUri]) resolvedStreamfiles[baseUri] = {};
+			resolvedStreamfiles[baseUri][baseString] = resolvedPath;
+		}
 
-	if (resolvedPath) {
-		if (!resolvedStreamfiles[baseUri]) resolvedStreamfiles[baseUri] = {};
-		resolvedStreamfiles[baseUri][baseString] = resolvedPath;
+		return resolvedPath;
+	} catch (e) {
+		console.log(`Streamfile resolve failed.`);
+		console.log(JSON.stringify({baseUri, base, paths}));
+		console.log(e);
 	}
 
-	return resolvedPath;
+	return undefined;
 }
 
 export function getWorkingDirectory(): Promise<string|undefined> {

extension/server/src/data.ts

@@ -55,6 +55,8 @@ export function dspffdToRecordFormats(data: any, aliases = false): Declaration[]
 			len: digits === 0 ? strLength : digits,
 			decimals: decimals,
 			keywords,
+			field: ``,
+			pos: ``
 		});
 		currentSubfield.description = text.trim();
 

extension/server/src/providers/definition.ts

@@ -5,18 +5,18 @@ import Cache from '../../../../language/models/cache';
 import Declaration from '../../../../language/models/declaration';
 
 export default async function definitionProvider(handler: DefinitionParams): Promise<Definition|null> {
-	const currentPath = handler.textDocument.uri;
+	const currentUri = handler.textDocument.uri;
 	const lineNumber = handler.position.line;
-	const document = documents.get(currentPath);
+	const document = documents.get(currentUri);
 
 	if (document) {
-		const doc = await parser.getDocs(currentPath, document.getText());
+		const doc = await parser.getDocs(currentUri, document.getText());
 		if (doc) {
 			const editingLine = document.getText(Range.create(lineNumber, 0, lineNumber, 200));
 			const possibleInclude = Parser.getIncludeFromDirective(editingLine);
 
-			if (possibleInclude) {
-				const include = await parser.includeFileFetch(currentPath, possibleInclude);
+			if (possibleInclude && parser.includeFileFetch) {
+				const include = await parser.includeFileFetch(currentUri, possibleInclude);
 				if (include.found && include.uri) {
 					return Location.create(include.uri, Range.create(0, 0, 0, 0));
 				}
@@ -25,7 +25,7 @@ export default async function definitionProvider(handler: DefinitionParams): Pro
 				let def: Declaration|undefined;
 
 				// First, we try and get the reference by offset
-				def = Cache.referenceByOffset(doc, document.offsetAt(handler.position));
+				def = Cache.referenceByOffset(currentUri, doc, document.offsetAt(handler.position));
 
 				if (def) {
 					return Location.create(

extension/server/src/providers/hover.ts

@@ -83,7 +83,9 @@ export default async function hoverProvider(params: HoverParams): Promise<Hover|
 
 				if (theVariable) {
 					// Variable definition found
-					let markdown = `\`${theVariable.name}\`: \`${prettyKeywords(theVariable.keyword)}\``;
+					const refs = theVariable.references.length;
+
+					let markdown = `\`${theVariable.name} ${prettyKeywords(theVariable.keyword)}\` (${refs} reference${refs === 1 ? `` : `s`})`;
 
 					if (theVariable.position && currentPath !== theVariable.position.path) {
 						markdown += `\n\n*@file* \`${theVariable.position.path}:${theVariable.position.line+1}\``;
@@ -101,7 +103,7 @@ export default async function hoverProvider(params: HoverParams): Promise<Hover|
 
 					const includeDirective = Parser.getIncludeFromDirective(lineContent);
 
-					if (includeDirective) {
+					if (includeDirective && parser.includeFileFetch) {
 						const include = await parser.includeFileFetch(currentPath, includeDirective);
 						let displayName = includeDirective;
 

extension/server/src/providers/linter/index.ts

@@ -48,7 +48,8 @@ export function initialise(connection: _Connection) {
 						document.getText(),
 						{
 							withIncludes: true,
-							ignoreCache: true
+							ignoreCache: true,
+							collectReferences: true
 						}
 					).then(cache => {
 						if (cache) {
@@ -198,7 +199,6 @@ export async function refreshLinterDiagnostics(document: TextDocument, docs: Cac
 
 		let availableIncludes: string[] | undefined;
 		if (Project.isEnabled) {
-			options.CollectReferences = true;
 			const headers = await Project.getIncludes(document.uri);
 			availableIncludes = headers.map(header => header.relative);
 		}
@@ -538,7 +538,7 @@ function caseInsensitiveReplaceAll(text: string, search: string, replace: string
 
 function createExtract(document: TextDocument, userRange: Range, docs: Cache) {
 	const range = Range.create(userRange.start.line, 0, userRange.end.line, 1000);
-	const references = docs.referencesInRange({position: document.offsetAt(range.start), end: document.offsetAt(range.end)});
+	const references = docs.referencesInRange(document.uri, {position: document.offsetAt(range.start), end: document.offsetAt(range.end)});
 	const validRefs = references.filter(ref => [`struct`, `subitem`, `variable`].includes(ref.dec.type));
 
 	const nameDiffSize = 1; // Always once since we only add 'p' at the start

extension/server/src/providers/project/index.ts

@@ -141,17 +141,7 @@ async function loadLocalFile(uri: string) {
 
 	if (document) {
 		const content = document?.getText();
-		const cache = await parser.getDocs(uri, content);
-		if (cache) {
-			if (content.length >= 6 && content.substring(0, 6).toUpperCase() === `**FREE`) {
-				Linter.getErrors({
-					uri,
-					content,
-				}, {
-					CollectReferences: true
-				}, cache);
-			}
-		}
+		await parser.getDocs(uri, content);
 	}
 }
 

extension/server/src/providers/project/references.ts

@@ -5,22 +5,11 @@ import { calculateOffset } from '../linter';
 import { documents, parser } from '..';
 import { getTextDoc, isEnabled } from '.';
 
-export async function findAllLocalReferences(def: Declaration): Promise<Location[]> {
+export async function findAllProjectReferences(def: Declaration): Promise<Location[]> {
 	let locations: Location[] = [];
 
 	if (isEnabled) {
 		const parsedFiles = Object.keys(parser.parsedCache);
-
-		const document = documents.get(def.position.path);
-
-		if (document) {
-			locations.push(
-				...def.references.map(ref => Location.create(
-					def.position.path,
-					calculateOffset(document, ref)
-				))
-			);
-		}
 
 		if (def.keyword[`EXPORT`]) {
 			// If we are looking for references to an export function
@@ -57,10 +46,6 @@ export async function findAllLocalReferences(def: Declaration): Promise<Location
 							// Don't add duplicates
 							if (!locations.some(loc => loc.uri === keyPath)) {
 								locations.push(
-									// First, we push the copybook where it is brought in.
-									// We do this because we don't have references for non-**free
-									Location.create(proc.position.path, Range.create(proc.position.line, 0, proc.position.line, 0)),
-
 									// Then we push the references. Empty for non-**free
 									...proc.references.map(ref => Location.create(
 										keyPath,
@@ -74,45 +59,6 @@ export async function findAllLocalReferences(def: Declaration): Promise<Location
 				}
 			}
 
-		} else {
-			// Otherwise, we are looking for references to the current definition
-			const baseUri = def.position.path;
-
-			for (const uri of parsedFiles) {
-				const document = await getTextDoc(uri);
-				if (document) {
-					const cache = parser.getParsedCache(uri);
-
-					// It's only a valid reference if it is imported somewhere else
-					const foundInclude = cache.includes.find(include => include.toPath === baseUri)
-					if (foundInclude) {
-						const possibleDef = cache.find(def.name);
-
-						// Okay, we found something with a similar name in another file...
-						if (possibleDef) {
-							if (possibleDef.position.path === def.position.path) {
-								if (document.getText(Range.create(0, 0, 0, 6)).toUpperCase() !== `**FREE` || possibleDef.references.length === 0) {
-									locations.push(
-										// First, we push the copybook where it is brought in.
-										// We do this because we don't have references for non-**free
-										Location.create(uri, Range.create(foundInclude.line, 0, foundInclude.line, 0)),
-									);
-								} else {
-									// But since it's **free, and we probably have referneces...
-									locations.push(
-										// Then we push the references. Empty for non-**free
-										...possibleDef.references.map(ref => Location.create(
-											uri,
-											calculateOffset(document, ref)
-										))
-									);
-								}
-
-							}
-						}
-					}
-				}
-			}
 		}
 	}
 

extension/server/src/providers/reference.ts

@@ -4,7 +4,7 @@ import Linter from '../../../../language/linter';
 import { calculateOffset } from './linter';
 
 import * as Project from "./project";
-import { findAllLocalReferences } from './project/references';
+import { findAllProjectReferences as getAllProcedureReferences } from './project/references';
 import Cache from '../../../../language/models/cache';
 
 export async function referenceProvider(params: ReferenceParams): Promise<Location[]|undefined> {
@@ -18,35 +18,29 @@ export async function referenceProvider(params: ReferenceParams): Promise<Locati
 		const document = documents.get(uri);
 
 		if (document) {
-			const isFree = (document.getText(Range.create(0, 0, 0, 6)).toUpperCase() === `**FREE`);
-
 			const doc = await parser.getDocs(uri, document.getText());
 
 			if (doc) {
-				if (isFree) {
-					Linter.getErrors(
-						{
-							uri,
-							content: document.getText()
-						},
-						{
-							CollectReferences: true
-						},
-						doc
-					);
-				}
-
-				const def = Cache.referenceByOffset(doc, document.offsetAt(currentPos));
+				const def = Cache.referenceByOffset(uri, doc, document.offsetAt(currentPos));
 
 				if (def) {
+					let locations: Location[] = [];
 					if (Project.isEnabled) {
-						return await findAllLocalReferences(def);
-					} else {
-						return def.references.map(ref => Location.create(
-							def.position.path,
-							calculateOffset(document, ref)
-						));	
+						const procRefs = await getAllProcedureReferences(def);
+						locations.push(...procRefs);
 					}
+
+					for (const ref of def.references) {
+						let refDoc = documents.get(ref.uri);
+						if (refDoc) {
+							locations.push(Location.create(
+								ref.uri,
+								calculateOffset(refDoc, ref)
+							));
+						}
+					}
+
+					return locations;
 				}
 			}
 		}