Merge pull request #2615 from fernewelten/Editor_NewScriptStart

Honor new-script markers in the preprocessor

Compiler/preproc/preprocessor.cpp

@@ -13,6 +13,7 @@
 //=============================================================================
 #include <algorithm>
 #include <cctype>
+#include <string.h>
 #include "script/cs_parser_common.h"
 #include "preproc/preprocessor.h"
 #include "script/cc_common.h"
@@ -24,7 +25,6 @@
 
 using namespace AGS::Common;
 
-extern int currentline; // in script/script_common
 
 namespace AGS {
 namespace Preprocessor {
@@ -148,6 +148,10 @@ namespace Preprocessor {
         err.Message = msg;
         _errors.push_back(err);
 
+        // 'cc_error()' will only work properly when the global variables
+        // 'currentline' and 'ccCurScriptName' are current
+        currentline = _lineNumber;
+        ccCurScriptName = _scriptName.GetCStr();
         cc_error(msg.GetCStr());
     }
 
@@ -245,7 +249,8 @@ namespace Preprocessor {
                     size_t endOfString = FindIndexOfMatchingCharacter(text, i, text[i]);
                     if (endOfString == NOT_FOUND) //size_t is unsigned but it's alright
                     {
-                        LogError(ErrorCode::UnterminatedString, "Unterminated string");
+                        String msg = String::FromFormat("Unterminated string: '%c' is missing", text[i]);
+                        LogError(ErrorCode::UnterminatedString, msg);
                         break;
                     }
                     endOfString++;
@@ -405,12 +410,24 @@ namespace Preprocessor {
             {
                 if ((line[i] == '"') || (line[i] == '\''))
                 {
-                    i = FindIndexOfMatchingCharacter(line, i, line[i]);
-                    if (i == NOT_FOUND)
+                    int end_of_literal = FindIndexOfMatchingCharacter(line, i, line[i]);
+                    if (end_of_literal == NOT_FOUND)
                     {
                         i = line.GetLength();
                         break;
                     }
+                    if (i == 0u && line[0u] == '"')
+                    {
+                        // '[end_of_literal]' contains the '"', we need the part before that
+                        String literal = line.Mid(0u, end_of_literal);
+                        if (literal.StartsWith(NEW_SCRIPT_TOKEN_PREFIX))
+                        {
+                            // Start the new script
+                            _scriptName = literal.Mid(strlen(NEW_SCRIPT_TOKEN_PREFIX));
+                            _lineNumber = 0;
+                        }
+                    }
+                    i = end_of_literal;
                 }
                 i++;
             }
@@ -463,15 +480,15 @@ namespace Preprocessor {
     {
         _errors.clear();
         StringBuilder output = StringBuilder(script.GetLength());
-        currentline = _lineNumber = 0;
+        _lineNumber = 0;
         String escapedScriptName = scriptName;
         escapedScriptName.Replace("\\", "\\\\");
         output.WriteLine(String::FromFormat("%s%s\"", NEW_SCRIPT_TOKEN_PREFIX, escapedScriptName.GetCStr()));
         StringReader reader = StringReader(script);
         _scriptName = scriptName;
         while (reader.IsValid())
         {
-            currentline = ++_lineNumber;
+            ++_lineNumber;
             String thisLine = reader.ReadLine();
             thisLine = RemoveComments(thisLine);
             if (thisLine.GetLength() > 0)
@@ -510,4 +527,4 @@ namespace Preprocessor {
     }
 
     } // Preprocessor
-} // AGS
+} // AGS

Compiler/preproc/preprocessor.h

@@ -82,4 +82,4 @@ namespace Preprocessor {
     };
 
 } // Preprocessor
-} // AGS
+} // AGS

Compiler/test/cc_test_helper.cpp

@@ -15,8 +15,6 @@
 #include "util/string_compat.h"
 #include "util/string.h"
 
-extern int currentline; // in script/script_common
-
 typedef AGS::Common::String AGSString;
 
 std::string last_cc_error_buf;

Compiler/test/cc_test_helper.h

@@ -25,5 +25,6 @@ extern void clear_error(void);
 extern const char *last_seen_cc_error(void);
 extern std::pair<AGS::Common::String, AGS::Common::String> cc_error_at_line(const char* error_msg);
 extern AGS::Common::String cc_error_without_line(const char* error_msg);
-
-#endif // __CC_TEST_HELPER_H
+extern int currentline;
+extern std::string ccCurScriptName;
+#endif // __CC_TEST_HELPER_H

Compiler/test/preprocessor_test.cpp

@@ -25,7 +25,7 @@ namespace Preprocessor {
 std::vector<AGSString> SplitLines(const AGSString& str)
 {
     std::vector<AGSString> str_lines = str.Split('\n');
-    for (int i = 0; i < str_lines.size(); i++) {
+    for (size_t i = 0; i < str_lines.size(); i++) {
         if (str_lines[i].CompareRight("\r", 1) == 0) {
             str_lines[i].ClipRight(1);
         }
@@ -812,5 +812,75 @@ void FuncИह€한𐍈() {
     ASSERT_EQ(pp.GetLastError().Type, ErrorCode::InvalidCharacter);
 }
 
+TEST(Preprocess, UnterminatedStringSingleQuote) {
+    Preprocessor pp = Preprocessor();
+    const char *inpl = R"EOS(
+int Func1()
+{
+    'unterminated string
+    return 0;
+}
+)EOS";
+
+    clear_error();
+    String res = pp.Preprocess(inpl, "UnterminatedStringSingleQuote");
+
+    // According to the googletest docs,
+    // the expected value should come first, the tested expression second
+    // Then in the case of a failed test, the reported message comes out correctly.
+    EXPECT_STREQ("Unterminated string: ''' is missing", last_seen_cc_error());
+    auto err = pp.GetLastError();
+    // script lines are 1-based, because line 0 is a NEW SCRIPT MARKER added by preproc
+    EXPECT_EQ(4, currentline);
+}
+
+TEST(Preprocess, UnterminatedStringDoubleQuote) {
+    Preprocessor pp = Preprocessor();
+    const char *inpl = R"EOS(
+int Func1()
+{
+    "unterminated string
+    return 0;
+}
+)EOS";
+
+    clear_error();
+    String res = pp.Preprocess(inpl, "UnterminatedStringDoubleQuote");
+
+    EXPECT_STREQ(last_seen_cc_error(), "Unterminated string: '\"' is missing");
+    // script lines are 1-based, because line 0 is a NEW SCRIPT MARKER added by preproc
+    EXPECT_EQ(currentline, 4);
+}
+
+TEST(Preprocess, MultipleNewScriptMarkers) {
+    Preprocessor pp = Preprocessor();
+    const char *inpl = R"EOS(
+"__NEWSCRIPTSTART_Dialog1"
+int Func1()
+{
+    return 0;
+}
+"__NEWSCRIPTSTART_Dialog2"
+int Func2()
+{
+    "unterminated string
+    return 0;
+}
+"__NEWSCRIPTSTART_Dialog3"
+int Func3()
+{
+    return 0;
+}
+)EOS";
+
+    clear_error();
+    String res = pp.Preprocess(inpl, "MultipleNewScriptMarkers");
+
+    EXPECT_STREQ("Unterminated string: '\"' is missing", last_seen_cc_error());
+    EXPECT_STREQ("Dialog2", ccCurScriptName.c_str());
+    // Line count starts from the nearest NEW SCRIPT MARKER
+    EXPECT_EQ(3, currentline);
+}
+
 } // Preprocessor
-} // AGS
+} // AGS

Editor/AGS.CScript.Compiler/Preprocessor.cs

@@ -99,14 +99,26 @@ private string PreProcessLine(string lineToProcess)
 				int i = 0;
 				while ((i < line.Length) && (!Char.IsLetterOrDigit(line[i])))
 				{
-					if ((line[i] == '"') || (line[i] == '\''))
-					{
-						i = FindIndexOfMatchingCharacter(line.ToString(), i, line[i]);
-						if (i < 0)
-						{
-							i = line.Length;
-							break;
-						}
+                    if ((line[i] == '"') || (line[i] == '\''))
+                    {
+                        int end_of_literal = FindIndexOfMatchingCharacter(line.ToString(), i, line[i]);
+                        if (end_of_literal < 0)
+                        {
+                            i = line.Length;
+                            break;
+                        }
+                        if (i == 0 && line[0] == '"')
+                        {
+                            // '[end_of_literal]' contains the '"', we need the part before that
+                            FastString literal = line.Substring(0, end_of_literal);
+                            if (literal.StartsWith(Constants.NEW_SCRIPT_MARKER))
+                            {
+                                // Start the new script
+                                _scriptName = literal.Substring(Constants.NEW_SCRIPT_MARKER.Length).ToString();
+                                _lineNumber = 0;
+                            }
+                        }
+                        i = end_of_literal;
 					}
 					i++;
 				}
@@ -388,7 +400,9 @@ private string RemoveComments(string text)
 						int endOfString = FindIndexOfMatchingCharacter(text, i, text[i]);
 						if (endOfString < 0)
 						{
-							RecordError(ErrorCode.UnterminatedString, "Unterminated string");
+                            RecordError(
+                                ErrorCode.UnterminatedString,
+                                $"Unterminated string: {text[i]} is missing");
 							break;
 						}
 						endOfString++;