From 7880b9d7a4f0be9da02f489268391cdce6921473 Mon Sep 17 00:00:00 2001 From: dgelessus Date: Sat, 29 Jul 2023 03:01:46 +0200 Subject: [PATCH 01/17] Add unit tests for console command tokenization --- .../pfConsoleCore/pfConsoleEngine.cpp | 28 +- .../pfConsoleCore/pfConsoleEngine.h | 2 + Sources/Tests/FeatureTests/CMakeLists.txt | 1 + .../pfConsoleCoreTest/CMakeLists.txt | 12 + .../pfConsoleCoreTest/test_pfConsoleCore.cpp | 409 ++++++++++++++++++ 5 files changed, 438 insertions(+), 14 deletions(-) create mode 100644 Sources/Tests/FeatureTests/pfConsoleCoreTest/CMakeLists.txt create mode 100644 Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp index b89b9a3575..7488f4ea9e 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp @@ -63,7 +63,7 @@ static const char kTokenSeparators[] = " =\r\n\t,"; static const char kTokenGrpSeps[] = " =\r\n._\t,"; //WARNING: Potentially increments the pointer passed to it. -static const char *console_strtok( char *&line, bool haveCommand ) +const char* pfConsoleEngine::Tokenize(char*& line, bool haveCommand) { char *begin = line; @@ -129,13 +129,13 @@ bool pfConsoleEngine::PrintCmdHelp(const ST::string& name, void (*PrintFn)(const pfConsoleCmdGroup *group, *subGrp; const char *ptr; - // console_strtok requires a writable C string... + // Tokenize requires a writable C string... ST::char_buffer nameBuf = name.to_utf8(); char* namePtr = nameBuf.data(); /// Scan for subgroups. This can be an empty loop group = pfConsoleCmdGroup::GetBaseGroup(); - ptr = console_strtok(namePtr, false); + ptr = Tokenize(namePtr, false); while (ptr != nullptr) { // Take this token and check to see if it's a group @@ -144,7 +144,7 @@ bool pfConsoleEngine::PrintCmdHelp(const ST::string& name, void (*PrintFn)(const else break; - ptr = console_strtok(namePtr, false); + ptr = Tokenize(namePtr, false); } if (ptr == nullptr) @@ -198,13 +198,13 @@ ST::string pfConsoleEngine::GetCmdSignature(const ST::string& name) pfConsoleCmdGroup *group, *subGrp; const char *ptr; - // console_strtok requires a writable C string... + // Tokenize requires a writable C string... ST::char_buffer nameBuf = name.to_utf8(); char* namePtr = nameBuf.data(); /// Scan for subgroups. This can be an empty loop group = pfConsoleCmdGroup::GetBaseGroup(); - ptr = console_strtok(namePtr, false); + ptr = Tokenize(namePtr, false); while (ptr != nullptr) { // Take this token and check to see if it's a group @@ -213,7 +213,7 @@ ST::string pfConsoleEngine::GetCmdSignature(const ST::string& name) else break; - ptr = console_strtok(namePtr, false); + ptr = Tokenize(namePtr, false); } if (ptr == nullptr) @@ -290,13 +290,13 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S const char *ptr; bool valid = true; - // console_strtok requires a writable C string... + // Tokenize requires a writable C string... ST::char_buffer lineBuf = line.to_utf8(); char* linePtr = lineBuf.data(); /// Loop #1: Scan for subgroups. This can be an empty loop group = pfConsoleCmdGroup::GetBaseGroup(); - ptr = console_strtok(linePtr, false); + ptr = Tokenize(linePtr, false); while (ptr != nullptr) { // Take this token and check to see if it's a group @@ -305,7 +305,7 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S else break; - ptr = console_strtok(linePtr, false); + ptr = Tokenize(linePtr, false); } if (ptr == nullptr) @@ -327,7 +327,7 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S /// params for( numParams = numQuotedParams = 0; numParams < fMaxNumParams - && (ptr = console_strtok(linePtr, true)) != nullptr + && (ptr = Tokenize(linePtr, true)) != nullptr && valid; numParams++ ) { if( ptr[ 0 ] == '\xFF' ) @@ -448,13 +448,13 @@ ST::string pfConsoleEngine::FindPartialCmd(const ST::string& line, bool findAgai /// New search ST::string_stream newStr; - // console_strtok requires a writable C string... + // Tokenize requires a writable C string... ST::char_buffer lineBuf = line.to_utf8(); char* linePtr = lineBuf.data(); /// Loop #1: Scan for subgroups. This can be an empty loop pfConsoleCmdGroup* group = pfConsoleCmdGroup::GetBaseGroup(); - const char* ptr = console_strtok(linePtr, false); + const char* ptr = Tokenize(linePtr, false); while (ptr != nullptr) { // Take this token and check to see if it's a group @@ -465,7 +465,7 @@ ST::string pfConsoleEngine::FindPartialCmd(const ST::string& line, bool findAgai group = subGrp; newStr << group->GetName() << '.'; - ptr = console_strtok(linePtr, false); + ptr = Tokenize(linePtr, false); } if (ptr != nullptr) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h index 0b3c25421b..92788c801d 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h @@ -81,6 +81,8 @@ class pfConsoleEngine pfConsoleEngine(); ~pfConsoleEngine(); + static const char* Tokenize(char*& line, bool haveCommand); + // Gets the signature for the command given (NO groups!) ST::string GetCmdSignature(const ST::string& name); diff --git a/Sources/Tests/FeatureTests/CMakeLists.txt b/Sources/Tests/FeatureTests/CMakeLists.txt index 05beac89e9..c42dd271e7 100644 --- a/Sources/Tests/FeatureTests/CMakeLists.txt +++ b/Sources/Tests/FeatureTests/CMakeLists.txt @@ -6,4 +6,5 @@ include_directories("${PLASMA_SOURCE_ROOT}/PubUtilLib/inc") include_directories("${PLASMA_SOURCE_ROOT}/FeatureLib") include_directories("${PLASMA_SOURCE_ROOT}/FeatureLib/inc") +add_subdirectory(pfConsoleCoreTest) add_subdirectory(pfPythonTest) diff --git a/Sources/Tests/FeatureTests/pfConsoleCoreTest/CMakeLists.txt b/Sources/Tests/FeatureTests/pfConsoleCoreTest/CMakeLists.txt new file mode 100644 index 0000000000..24d389a189 --- /dev/null +++ b/Sources/Tests/FeatureTests/pfConsoleCoreTest/CMakeLists.txt @@ -0,0 +1,12 @@ +set(pfConsoleCore_SOURCES + test_pfConsoleCore.cpp +) + +plasma_test(test_pfConsoleCore SOURCES ${pfConsoleCore_SOURCES}) +target_link_libraries( + test_pfConsoleCore + PRIVATE + CoreLib + pfConsoleCore + gtest_main +) diff --git a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp new file mode 100644 index 0000000000..5ac9ff1ea2 --- /dev/null +++ b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp @@ -0,0 +1,409 @@ +/*==LICENSE==* + +CyanWorlds.com Engine - MMOG client, server and tools +Copyright (C) 2011 Cyan Worlds, Inc. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Additional permissions under GNU GPL version 3 section 7 + +If you modify this Program, or any covered work, by linking or +combining it with any of RAD Game Tools Bink SDK, Autodesk 3ds Max SDK, +NVIDIA PhysX SDK, Microsoft DirectX SDK, OpenSSL library, Independent +JPEG Group JPEG library, Microsoft Windows Media SDK, or Apple QuickTime SDK +(or a modified version of those libraries), +containing parts covered by the terms of the Bink SDK EULA, 3ds Max EULA, +PhysX SDK EULA, DirectX SDK EULA, OpenSSL and SSLeay licenses, IJG +JPEG Library README, Windows Media SDK EULA, or QuickTime SDK EULA, the +licensors of this Program grant you additional +permission to convey the resulting work. Corresponding Source for a +non-source form of such a combination shall include the source code for +the parts of OpenSSL and IJG JPEG Library used as well as that of the covered +work. + +You can contact Cyan Worlds, Inc. by email legal@cyan.com + or by snail mail at: + Cyan Worlds, Inc. + 14617 N Newport Hwy + Mead, WA 99021 + +*==LICENSE==*/ + +#include + +#include "pfConsoleCore/pfConsoleEngine.h" + +// Tokenize command name, 1 token + +TEST(pfConsoleCore, TokenizeCommandNameSingle) +{ + char buf[] = "SampleCmd1"; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, false); + EXPECT_STREQ(token1, "SampleCmd1"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, false); + EXPECT_EQ(token2, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +TEST(pfConsoleCore, TokenizeCommandNameSingleWhitespace) +{ + char buf[] = " SampleCmd1 "; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, false); + EXPECT_STREQ(token1, "SampleCmd1"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, false); + EXPECT_EQ(token2, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +// Tokenize command name, 2 tokens + +TEST(pfConsoleCore, TokenizeCommandNameDot) +{ + char buf[] = "App.Quit"; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, false); + EXPECT_STREQ(token1, "App"); + EXPECT_EQ(line, buf + sizeof("App.") - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, false); + EXPECT_STREQ(token2, "Quit"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token3 = pfConsoleEngine::Tokenize(line, false); + EXPECT_EQ(token3, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +TEST(pfConsoleCore, TokenizeCommandNameUnderscore) +{ + char buf[] = "App_Quit"; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, false); + EXPECT_STREQ(token1, "App"); + EXPECT_EQ(line, buf + sizeof("App_") - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, false); + EXPECT_STREQ(token2, "Quit"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token3 = pfConsoleEngine::Tokenize(line, false); + EXPECT_EQ(token3, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +TEST(pfConsoleCore, TokenizeCommandNameSpace) +{ + char buf[] = "App Quit"; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, false); + EXPECT_STREQ(token1, "App"); + EXPECT_EQ(line, buf + sizeof("App ") - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, false); + EXPECT_STREQ(token2, "Quit"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token3 = pfConsoleEngine::Tokenize(line, false); + EXPECT_EQ(token3, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +// Tokenize command name, 3 tokens + +TEST(pfConsoleCore, TokenizeCommandNameDots) +{ + char buf[] = "Graphics.Renderer.SetYon"; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, false); + EXPECT_STREQ(token1, "Graphics"); + EXPECT_EQ(line, buf + sizeof("Graphics.") - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, false); + EXPECT_STREQ(token2, "Renderer"); + EXPECT_EQ(line, buf + sizeof("Graphics.Renderer.") - 1); + + const char* token3 = pfConsoleEngine::Tokenize(line, false); + EXPECT_STREQ(token3, "SetYon"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token4 = pfConsoleEngine::Tokenize(line, false); + EXPECT_EQ(token4, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +TEST(pfConsoleCore, TokenizeCommandNameUnderscores) +{ + char buf[] = "Graphics_Renderer_SetYon"; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, false); + EXPECT_STREQ(token1, "Graphics"); + EXPECT_EQ(line, buf + sizeof("Graphics_") - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, false); + EXPECT_STREQ(token2, "Renderer"); + EXPECT_EQ(line, buf + sizeof("Graphics_Renderer_") - 1); + + const char* token3 = pfConsoleEngine::Tokenize(line, false); + EXPECT_STREQ(token3, "SetYon"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token4 = pfConsoleEngine::Tokenize(line, false); + EXPECT_EQ(token4, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +TEST(pfConsoleCore, TokenizeCommandNameSpaces) +{ + char buf[] = "Graphics Renderer SetYon"; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, false); + EXPECT_STREQ(token1, "Graphics"); + EXPECT_EQ(line, buf + sizeof("Graphics ") - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, false); + EXPECT_STREQ(token2, "Renderer"); + EXPECT_EQ(line, buf + sizeof("Graphics Renderer ") - 1); + + const char* token3 = pfConsoleEngine::Tokenize(line, false); + EXPECT_STREQ(token3, "SetYon"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token4 = pfConsoleEngine::Tokenize(line, false); + EXPECT_EQ(token4, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +// Tokenize arguments, 1 token + +TEST(pfConsoleCore, TokenizeArgumentsSingle) +{ + char buf[] = "arg"; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token1, "arg"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, true); + EXPECT_EQ(token2, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +TEST(pfConsoleCore, TokenizeArgumentsSingleWhitespace) +{ + char buf[] = " arg "; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token1, "arg"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, true); + EXPECT_EQ(token2, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +TEST(pfConsoleCore, TokenizeArgumentsSingleUnderscore) +{ + char buf[] = "arg_test"; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token1, "arg_test"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, true); + EXPECT_EQ(token2, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +TEST(pfConsoleCore, TokenizeArgumentsSingleDoubleQuote) +{ + char buf[] = "\"(Default Device)\""; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token1, "(Default Device)"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, true); + EXPECT_EQ(token2, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +TEST(pfConsoleCore, TokenizeArgumentsSingleSingleQuote) +{ + char buf[] = "'(Default Device)'"; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token1, "(Default Device)"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, true); + EXPECT_EQ(token2, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +TEST(pfConsoleCore, TokenizeArgumentsSingleDoubleQuoteUnclosed) +{ + char buf[] = "\"(Default Device)"; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token1, "\xff"); +} + +TEST(pfConsoleCore, TokenizeArgumentsSingleSingleQuoteUnclosed) +{ + char buf[] = "'(Default Device)"; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token1, "\xff"); +} + +// Tokenize arguments, 2 tokens + +TEST(pfConsoleCore, TokenizeArgumentsPair) +{ + char buf[] = "arg1 arg2"; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token1, "arg1"); + EXPECT_EQ(line, buf + sizeof("arg1 ") - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token2, "arg2"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token3 = pfConsoleEngine::Tokenize(line, true); + EXPECT_EQ(token3, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +TEST(pfConsoleCore, TokenizeArgumentsPairWhitespace) +{ + char buf[] = " arg1 arg2 "; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token1, "arg1"); + EXPECT_EQ(line, buf + sizeof(" arg1 ") - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token2, "arg2"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token3 = pfConsoleEngine::Tokenize(line, true); + EXPECT_EQ(token3, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +TEST(pfConsoleCore, TokenizeArgumentsPairComma) +{ + char buf[] = "arg1, arg2"; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token1, "arg1"); + EXPECT_EQ(line, buf + sizeof("arg1,") - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token2, "arg2"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token3 = pfConsoleEngine::Tokenize(line, true); + EXPECT_EQ(token3, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +TEST(pfConsoleCore, TokenizeArgumentsPairMixedQuotes) +{ + char buf[] = "\"argument '1'\" 'argument \"2\"'"; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token1, "argument '1'"); + EXPECT_EQ(line, buf + sizeof("\"argument '1'\"") - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token2, "argument \"2\""); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token3 = pfConsoleEngine::Tokenize(line, true); + EXPECT_EQ(token3, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +// Tokenize arguments, 3 tokens + +TEST(pfConsoleCore, TokenizeArgumentsTriple) +{ + char buf[] = "1.2 3.4 5.6"; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token1, "1.2"); + EXPECT_EQ(line, buf + sizeof("1.2 ") - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token2, "3.4"); + EXPECT_EQ(line, buf + sizeof("1.2 3.4 ") - 1); + + const char* token3 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token3, "5.6"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token4 = pfConsoleEngine::Tokenize(line, true); + EXPECT_EQ(token4, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} + +TEST(pfConsoleCore, TokenizeArgumentsTripleCommas) +{ + char buf[] = "1.2, 3.4, 5.6"; + char* line = buf; + + const char* token1 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token1, "1.2"); + EXPECT_EQ(line, buf + sizeof("1.2,") - 1); + + const char* token2 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token2, "3.4"); + EXPECT_EQ(line, buf + sizeof("1.2, 3.4,") - 1); + + const char* token3 = pfConsoleEngine::Tokenize(line, true); + EXPECT_STREQ(token3, "5.6"); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + const char* token4 = pfConsoleEngine::Tokenize(line, true); + EXPECT_EQ(token4, nullptr); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} From 7992bd07e38216136ffb080c74d01ed7087688c9 Mon Sep 17 00:00:00 2001 From: dgelessus Date: Sat, 29 Jul 2023 03:11:50 +0200 Subject: [PATCH 02/17] Split pfConsoleEngine tokenization of command names and arguments Although they share a bit of code, the core logic is quite different. --- .../pfConsoleCore/pfConsoleEngine.cpp | 95 +++++++++------ .../pfConsoleCore/pfConsoleEngine.h | 3 +- .../pfConsoleCoreTest/test_pfConsoleCore.cpp | 114 +++++++++--------- 3 files changed, 114 insertions(+), 98 deletions(-) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp index 7488f4ea9e..7b126852f2 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp @@ -63,7 +63,7 @@ static const char kTokenSeparators[] = " =\r\n\t,"; static const char kTokenGrpSeps[] = " =\r\n._\t,"; //WARNING: Potentially increments the pointer passed to it. -const char* pfConsoleEngine::Tokenize(char*& line, bool haveCommand) +const char* pfConsoleEngine::TokenizeCommandName(char*& line) { char *begin = line; @@ -71,34 +71,49 @@ const char* pfConsoleEngine::Tokenize(char*& line, bool haveCommand) ++begin; for (line = begin; *line; ++line) { - if (!haveCommand) { - for (const char *sep = kTokenGrpSeps; *sep; ++sep) { - if (*line == *sep) { - *line = 0; - while (*++line && (*line == *sep)) - /* skip duplicate delimiters */; - return begin; - } + for (const char *sep = kTokenGrpSeps; *sep; ++sep) { + if (*line == *sep) { + *line = 0; + while (*++line && (*line == *sep)) + /* skip duplicate delimiters */; + return begin; } - } else { - if (*begin == '"' || *begin == '\'') { - // Handle strings as a single token - char *endptr = strchr(line + 1, *line); - if (endptr == nullptr) { - // Bad string token sentry - return "\xFF"; - } - *endptr = 0; - line = endptr + 1; - return begin + 1; + } + } + + if (begin == line) + return nullptr; + + line = line + strlen(line); + return begin; +} + +//WARNING: Potentially increments the pointer passed to it. +const char* pfConsoleEngine::TokenizeArguments(char*& line) +{ + char *begin = line; + + while (*begin && isspace(static_cast(*begin))) + ++begin; + + for (line = begin; *line; ++line) { + if (*begin == '"' || *begin == '\'') { + // Handle strings as a single token + char *endptr = strchr(line + 1, *line); + if (endptr == nullptr) { + // Bad string token sentry + return "\xFF"; } - for (const char *sep = kTokenSeparators; *sep; ++sep) { - if (*line == *sep) { - *line = 0; - while (*++line && (*line == *sep)) - /* skip duplicate delimiters */; - return begin; - } + *endptr = 0; + line = endptr + 1; + return begin + 1; + } + for (const char *sep = kTokenSeparators; *sep; ++sep) { + if (*line == *sep) { + *line = 0; + while (*++line && (*line == *sep)) + /* skip duplicate delimiters */; + return begin; } } } @@ -129,13 +144,13 @@ bool pfConsoleEngine::PrintCmdHelp(const ST::string& name, void (*PrintFn)(const pfConsoleCmdGroup *group, *subGrp; const char *ptr; - // Tokenize requires a writable C string... + // TokenizeCommandName requires a writable C string... ST::char_buffer nameBuf = name.to_utf8(); char* namePtr = nameBuf.data(); /// Scan for subgroups. This can be an empty loop group = pfConsoleCmdGroup::GetBaseGroup(); - ptr = Tokenize(namePtr, false); + ptr = TokenizeCommandName(namePtr); while (ptr != nullptr) { // Take this token and check to see if it's a group @@ -144,7 +159,7 @@ bool pfConsoleEngine::PrintCmdHelp(const ST::string& name, void (*PrintFn)(const else break; - ptr = Tokenize(namePtr, false); + ptr = TokenizeCommandName(namePtr); } if (ptr == nullptr) @@ -198,13 +213,13 @@ ST::string pfConsoleEngine::GetCmdSignature(const ST::string& name) pfConsoleCmdGroup *group, *subGrp; const char *ptr; - // Tokenize requires a writable C string... + // TokenizeCommandName requires a writable C string... ST::char_buffer nameBuf = name.to_utf8(); char* namePtr = nameBuf.data(); /// Scan for subgroups. This can be an empty loop group = pfConsoleCmdGroup::GetBaseGroup(); - ptr = Tokenize(namePtr, false); + ptr = TokenizeCommandName(namePtr); while (ptr != nullptr) { // Take this token and check to see if it's a group @@ -213,7 +228,7 @@ ST::string pfConsoleEngine::GetCmdSignature(const ST::string& name) else break; - ptr = Tokenize(namePtr, false); + ptr = TokenizeCommandName(namePtr); } if (ptr == nullptr) @@ -290,13 +305,13 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S const char *ptr; bool valid = true; - // Tokenize requires a writable C string... + // TokenizeCommandName/TokenizeArguments requires a writable C string... ST::char_buffer lineBuf = line.to_utf8(); char* linePtr = lineBuf.data(); /// Loop #1: Scan for subgroups. This can be an empty loop group = pfConsoleCmdGroup::GetBaseGroup(); - ptr = Tokenize(linePtr, false); + ptr = TokenizeCommandName(linePtr); while (ptr != nullptr) { // Take this token and check to see if it's a group @@ -305,7 +320,7 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S else break; - ptr = Tokenize(linePtr, false); + ptr = TokenizeCommandName(linePtr); } if (ptr == nullptr) @@ -327,7 +342,7 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S /// params for( numParams = numQuotedParams = 0; numParams < fMaxNumParams - && (ptr = Tokenize(linePtr, true)) != nullptr + && (ptr = TokenizeArguments(linePtr)) != nullptr && valid; numParams++ ) { if( ptr[ 0 ] == '\xFF' ) @@ -448,13 +463,13 @@ ST::string pfConsoleEngine::FindPartialCmd(const ST::string& line, bool findAgai /// New search ST::string_stream newStr; - // Tokenize requires a writable C string... + // TokenizeCommandName requires a writable C string... ST::char_buffer lineBuf = line.to_utf8(); char* linePtr = lineBuf.data(); /// Loop #1: Scan for subgroups. This can be an empty loop pfConsoleCmdGroup* group = pfConsoleCmdGroup::GetBaseGroup(); - const char* ptr = Tokenize(linePtr, false); + const char* ptr = TokenizeCommandName(linePtr); while (ptr != nullptr) { // Take this token and check to see if it's a group @@ -465,7 +480,7 @@ ST::string pfConsoleEngine::FindPartialCmd(const ST::string& line, bool findAgai group = subGrp; newStr << group->GetName() << '.'; - ptr = Tokenize(linePtr, false); + ptr = TokenizeCommandName(linePtr); } if (ptr != nullptr) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h index 92788c801d..ab75a81bf5 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h @@ -81,7 +81,8 @@ class pfConsoleEngine pfConsoleEngine(); ~pfConsoleEngine(); - static const char* Tokenize(char*& line, bool haveCommand); + static const char* TokenizeCommandName(char*& line); + static const char* TokenizeArguments(char*& line); // Gets the signature for the command given (NO groups!) ST::string GetCmdSignature(const ST::string& name); diff --git a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp index 5ac9ff1ea2..5967af9fa0 100644 --- a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp +++ b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp @@ -51,11 +51,11 @@ TEST(pfConsoleCore, TokenizeCommandNameSingle) char buf[] = "SampleCmd1"; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, false); + const char* token1 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_STREQ(token1, "SampleCmd1"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, false); + const char* token2 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_EQ(token2, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -65,11 +65,11 @@ TEST(pfConsoleCore, TokenizeCommandNameSingleWhitespace) char buf[] = " SampleCmd1 "; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, false); + const char* token1 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_STREQ(token1, "SampleCmd1"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, false); + const char* token2 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_EQ(token2, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -81,15 +81,15 @@ TEST(pfConsoleCore, TokenizeCommandNameDot) char buf[] = "App.Quit"; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, false); + const char* token1 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_STREQ(token1, "App"); EXPECT_EQ(line, buf + sizeof("App.") - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, false); + const char* token2 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_STREQ(token2, "Quit"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token3 = pfConsoleEngine::Tokenize(line, false); + const char* token3 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_EQ(token3, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -99,15 +99,15 @@ TEST(pfConsoleCore, TokenizeCommandNameUnderscore) char buf[] = "App_Quit"; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, false); + const char* token1 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_STREQ(token1, "App"); EXPECT_EQ(line, buf + sizeof("App_") - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, false); + const char* token2 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_STREQ(token2, "Quit"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token3 = pfConsoleEngine::Tokenize(line, false); + const char* token3 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_EQ(token3, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -117,15 +117,15 @@ TEST(pfConsoleCore, TokenizeCommandNameSpace) char buf[] = "App Quit"; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, false); + const char* token1 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_STREQ(token1, "App"); EXPECT_EQ(line, buf + sizeof("App ") - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, false); + const char* token2 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_STREQ(token2, "Quit"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token3 = pfConsoleEngine::Tokenize(line, false); + const char* token3 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_EQ(token3, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -137,19 +137,19 @@ TEST(pfConsoleCore, TokenizeCommandNameDots) char buf[] = "Graphics.Renderer.SetYon"; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, false); + const char* token1 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_STREQ(token1, "Graphics"); EXPECT_EQ(line, buf + sizeof("Graphics.") - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, false); + const char* token2 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_STREQ(token2, "Renderer"); EXPECT_EQ(line, buf + sizeof("Graphics.Renderer.") - 1); - const char* token3 = pfConsoleEngine::Tokenize(line, false); + const char* token3 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_STREQ(token3, "SetYon"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token4 = pfConsoleEngine::Tokenize(line, false); + const char* token4 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_EQ(token4, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -159,19 +159,19 @@ TEST(pfConsoleCore, TokenizeCommandNameUnderscores) char buf[] = "Graphics_Renderer_SetYon"; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, false); + const char* token1 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_STREQ(token1, "Graphics"); EXPECT_EQ(line, buf + sizeof("Graphics_") - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, false); + const char* token2 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_STREQ(token2, "Renderer"); EXPECT_EQ(line, buf + sizeof("Graphics_Renderer_") - 1); - const char* token3 = pfConsoleEngine::Tokenize(line, false); + const char* token3 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_STREQ(token3, "SetYon"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token4 = pfConsoleEngine::Tokenize(line, false); + const char* token4 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_EQ(token4, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -181,19 +181,19 @@ TEST(pfConsoleCore, TokenizeCommandNameSpaces) char buf[] = "Graphics Renderer SetYon"; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, false); + const char* token1 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_STREQ(token1, "Graphics"); EXPECT_EQ(line, buf + sizeof("Graphics ") - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, false); + const char* token2 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_STREQ(token2, "Renderer"); EXPECT_EQ(line, buf + sizeof("Graphics Renderer ") - 1); - const char* token3 = pfConsoleEngine::Tokenize(line, false); + const char* token3 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_STREQ(token3, "SetYon"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token4 = pfConsoleEngine::Tokenize(line, false); + const char* token4 = pfConsoleEngine::TokenizeCommandName(line); EXPECT_EQ(token4, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -205,11 +205,11 @@ TEST(pfConsoleCore, TokenizeArgumentsSingle) char buf[] = "arg"; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, true); + const char* token1 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token1, "arg"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, true); + const char* token2 = pfConsoleEngine::TokenizeArguments(line); EXPECT_EQ(token2, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -219,11 +219,11 @@ TEST(pfConsoleCore, TokenizeArgumentsSingleWhitespace) char buf[] = " arg "; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, true); + const char* token1 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token1, "arg"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, true); + const char* token2 = pfConsoleEngine::TokenizeArguments(line); EXPECT_EQ(token2, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -233,11 +233,11 @@ TEST(pfConsoleCore, TokenizeArgumentsSingleUnderscore) char buf[] = "arg_test"; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, true); + const char* token1 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token1, "arg_test"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, true); + const char* token2 = pfConsoleEngine::TokenizeArguments(line); EXPECT_EQ(token2, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -247,11 +247,11 @@ TEST(pfConsoleCore, TokenizeArgumentsSingleDoubleQuote) char buf[] = "\"(Default Device)\""; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, true); + const char* token1 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token1, "(Default Device)"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, true); + const char* token2 = pfConsoleEngine::TokenizeArguments(line); EXPECT_EQ(token2, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -261,11 +261,11 @@ TEST(pfConsoleCore, TokenizeArgumentsSingleSingleQuote) char buf[] = "'(Default Device)'"; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, true); + const char* token1 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token1, "(Default Device)"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, true); + const char* token2 = pfConsoleEngine::TokenizeArguments(line); EXPECT_EQ(token2, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -275,7 +275,7 @@ TEST(pfConsoleCore, TokenizeArgumentsSingleDoubleQuoteUnclosed) char buf[] = "\"(Default Device)"; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, true); + const char* token1 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token1, "\xff"); } @@ -284,7 +284,7 @@ TEST(pfConsoleCore, TokenizeArgumentsSingleSingleQuoteUnclosed) char buf[] = "'(Default Device)"; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, true); + const char* token1 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token1, "\xff"); } @@ -295,15 +295,15 @@ TEST(pfConsoleCore, TokenizeArgumentsPair) char buf[] = "arg1 arg2"; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, true); + const char* token1 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token1, "arg1"); EXPECT_EQ(line, buf + sizeof("arg1 ") - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, true); + const char* token2 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token2, "arg2"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token3 = pfConsoleEngine::Tokenize(line, true); + const char* token3 = pfConsoleEngine::TokenizeArguments(line); EXPECT_EQ(token3, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -313,15 +313,15 @@ TEST(pfConsoleCore, TokenizeArgumentsPairWhitespace) char buf[] = " arg1 arg2 "; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, true); + const char* token1 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token1, "arg1"); EXPECT_EQ(line, buf + sizeof(" arg1 ") - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, true); + const char* token2 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token2, "arg2"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token3 = pfConsoleEngine::Tokenize(line, true); + const char* token3 = pfConsoleEngine::TokenizeArguments(line); EXPECT_EQ(token3, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -331,15 +331,15 @@ TEST(pfConsoleCore, TokenizeArgumentsPairComma) char buf[] = "arg1, arg2"; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, true); + const char* token1 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token1, "arg1"); EXPECT_EQ(line, buf + sizeof("arg1,") - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, true); + const char* token2 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token2, "arg2"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token3 = pfConsoleEngine::Tokenize(line, true); + const char* token3 = pfConsoleEngine::TokenizeArguments(line); EXPECT_EQ(token3, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -349,15 +349,15 @@ TEST(pfConsoleCore, TokenizeArgumentsPairMixedQuotes) char buf[] = "\"argument '1'\" 'argument \"2\"'"; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, true); + const char* token1 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token1, "argument '1'"); EXPECT_EQ(line, buf + sizeof("\"argument '1'\"") - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, true); + const char* token2 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token2, "argument \"2\""); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token3 = pfConsoleEngine::Tokenize(line, true); + const char* token3 = pfConsoleEngine::TokenizeArguments(line); EXPECT_EQ(token3, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -369,19 +369,19 @@ TEST(pfConsoleCore, TokenizeArgumentsTriple) char buf[] = "1.2 3.4 5.6"; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, true); + const char* token1 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token1, "1.2"); EXPECT_EQ(line, buf + sizeof("1.2 ") - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, true); + const char* token2 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token2, "3.4"); EXPECT_EQ(line, buf + sizeof("1.2 3.4 ") - 1); - const char* token3 = pfConsoleEngine::Tokenize(line, true); + const char* token3 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token3, "5.6"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token4 = pfConsoleEngine::Tokenize(line, true); + const char* token4 = pfConsoleEngine::TokenizeArguments(line); EXPECT_EQ(token4, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -391,19 +391,19 @@ TEST(pfConsoleCore, TokenizeArgumentsTripleCommas) char buf[] = "1.2, 3.4, 5.6"; char* line = buf; - const char* token1 = pfConsoleEngine::Tokenize(line, true); + const char* token1 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token1, "1.2"); EXPECT_EQ(line, buf + sizeof("1.2,") - 1); - const char* token2 = pfConsoleEngine::Tokenize(line, true); + const char* token2 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token2, "3.4"); EXPECT_EQ(line, buf + sizeof("1.2, 3.4,") - 1); - const char* token3 = pfConsoleEngine::Tokenize(line, true); + const char* token3 = pfConsoleEngine::TokenizeArguments(line); EXPECT_STREQ(token3, "5.6"); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token4 = pfConsoleEngine::Tokenize(line, true); + const char* token4 = pfConsoleEngine::TokenizeArguments(line); EXPECT_EQ(token4, nullptr); EXPECT_EQ(line, buf + sizeof(buf) - 1); } From a906d76f50e75a14b950b9a554e88600c12c10a4 Mon Sep 17 00:00:00 2001 From: dgelessus Date: Sat, 29 Jul 2023 03:33:05 +0200 Subject: [PATCH 03/17] Remove pointless line adjustment at end of pfConsoleEngine tokenization This code could only be reached when *line == '\0', so strlen(line) was always 0 and the addition did nothing. --- Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp index 7b126852f2..df656c38f7 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp @@ -84,7 +84,6 @@ const char* pfConsoleEngine::TokenizeCommandName(char*& line) if (begin == line) return nullptr; - line = line + strlen(line); return begin; } @@ -121,7 +120,6 @@ const char* pfConsoleEngine::TokenizeArguments(char*& line) if (begin == line) return nullptr; - line = line + strlen(line); return begin; } From 0ec7972150e989a1ee594b8ebcbc8c56db03bbab Mon Sep 17 00:00:00 2001 From: dgelessus Date: Sat, 29 Jul 2023 03:43:33 +0200 Subject: [PATCH 04/17] Remove unused numQuotedParams in pfConsoleEngine::RunCommand --- Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp index df656c38f7..7d8f4914ed 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp @@ -298,7 +298,7 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S { pfConsoleCmd *cmd; pfConsoleCmdGroup *group, *subGrp; - int32_t numParams, i, numQuotedParams = 0; + int32_t numParams, i; pfConsoleCmdParam paramArray[ fMaxNumParams + 1 ]; const char *ptr; bool valid = true; @@ -339,7 +339,7 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S /// tokenizing (with the new separators now, mind you) and turn them into /// params - for( numParams = numQuotedParams = 0; numParams < fMaxNumParams + for (numParams = 0; numParams < fMaxNumParams && (ptr = TokenizeArguments(linePtr)) != nullptr && valid; numParams++ ) { From f6e577dc2e97d368cfedbd1cecf000adcf199952 Mon Sep 17 00:00:00 2001 From: dgelessus Date: Sat, 29 Jul 2023 15:32:26 +0200 Subject: [PATCH 05/17] Port pfConsoleEngine tokenization return values to ST::string This way the input buffer no longer needs to be modified, because the returned tokens are copied and not returned "in-place" from the buffer. --- .../pfConsoleCore/pfConsoleEngine.cpp | 133 ++++---- .../pfConsoleCore/pfConsoleEngine.h | 15 +- .../pfConsoleCoreTest/test_pfConsoleCore.cpp | 314 +++++++++--------- 3 files changed, 227 insertions(+), 235 deletions(-) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp index 7d8f4914ed..6fff6dfa12 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp @@ -58,14 +58,16 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com const int32_t pfConsoleEngine::fMaxNumParams = 16; +// Yes, this is invalid UTF-8. Yes, this is awful. +const ST::string pfConsoleEngine::kTokenizeError = ST_LITERAL("\xff"); static const char kTokenSeparators[] = " =\r\n\t,"; static const char kTokenGrpSeps[] = " =\r\n._\t,"; //WARNING: Potentially increments the pointer passed to it. -const char* pfConsoleEngine::TokenizeCommandName(char*& line) +std::optional pfConsoleEngine::TokenizeCommandName(const char*& line) { - char *begin = line; + const char* begin = line; while (*begin && isspace(static_cast(*begin))) ++begin; @@ -73,24 +75,25 @@ const char* pfConsoleEngine::TokenizeCommandName(char*& line) for (line = begin; *line; ++line) { for (const char *sep = kTokenGrpSeps; *sep; ++sep) { if (*line == *sep) { - *line = 0; + const char* end = line; while (*++line && (*line == *sep)) /* skip duplicate delimiters */; - return begin; + return ST::string::from_utf8(begin, end - begin); } } } - if (begin == line) - return nullptr; + if (begin == line) { + return {}; + } return begin; } //WARNING: Potentially increments the pointer passed to it. -const char* pfConsoleEngine::TokenizeArguments(char*& line) +std::optional pfConsoleEngine::TokenizeArguments(const char*& line) { - char *begin = line; + const char* begin = line; while (*begin && isspace(static_cast(*begin))) ++begin; @@ -98,27 +101,27 @@ const char* pfConsoleEngine::TokenizeArguments(char*& line) for (line = begin; *line; ++line) { if (*begin == '"' || *begin == '\'') { // Handle strings as a single token - char *endptr = strchr(line + 1, *line); - if (endptr == nullptr) { - // Bad string token sentry - return "\xFF"; + ++begin; + const char* end = strchr(begin, *line); + if (end == nullptr) { + return kTokenizeError; } - *endptr = 0; - line = endptr + 1; - return begin + 1; + line = end + 1; + return ST::string::from_utf8(begin, end - begin); } for (const char *sep = kTokenSeparators; *sep; ++sep) { if (*line == *sep) { - *line = 0; + const char* end = line; while (*++line && (*line == *sep)) /* skip duplicate delimiters */; - return begin; + return ST::string::from_utf8(begin, end - begin); } } } - if (begin == line) - return nullptr; + if (begin == line) { + return {}; + } return begin; } @@ -140,28 +143,22 @@ bool pfConsoleEngine::PrintCmdHelp(const ST::string& name, void (*PrintFn)(const { pfConsoleCmd *cmd; pfConsoleCmdGroup *group, *subGrp; - const char *ptr; - - // TokenizeCommandName requires a writable C string... - ST::char_buffer nameBuf = name.to_utf8(); - char* namePtr = nameBuf.data(); + const char* namePtr = name.c_str(); /// Scan for subgroups. This can be an empty loop group = pfConsoleCmdGroup::GetBaseGroup(); - ptr = TokenizeCommandName(namePtr); - while (ptr != nullptr) - { + auto token = TokenizeCommandName(namePtr); + while (token) { // Take this token and check to see if it's a group - if ((subGrp = group->FindSubGroupNoCase(ptr)) != nullptr) + if ((subGrp = group->FindSubGroupNoCase(*token)) != nullptr) group = subGrp; else break; - ptr = TokenizeCommandName(namePtr); + token = TokenizeCommandName(namePtr); } - if (ptr == nullptr) - { + if (!token) { if (group == nullptr) { fErrorMsg = ST_LITERAL("Invalid command syntax"); @@ -188,7 +185,7 @@ bool pfConsoleEngine::PrintCmdHelp(const ST::string& name, void (*PrintFn)(const } /// OK, so what we found wasn't a group. Which means we need a command... - cmd = group->FindCommandNoCase( ptr ); + cmd = group->FindCommandNoCase(*token); if (cmd == nullptr) { fErrorMsg = ST_LITERAL("Invalid syntax: command not found"); @@ -209,34 +206,28 @@ ST::string pfConsoleEngine::GetCmdSignature(const ST::string& name) { pfConsoleCmd *cmd; pfConsoleCmdGroup *group, *subGrp; - const char *ptr; - - // TokenizeCommandName requires a writable C string... - ST::char_buffer nameBuf = name.to_utf8(); - char* namePtr = nameBuf.data(); + const char* namePtr = name.c_str(); /// Scan for subgroups. This can be an empty loop group = pfConsoleCmdGroup::GetBaseGroup(); - ptr = TokenizeCommandName(namePtr); - while (ptr != nullptr) - { + auto token = TokenizeCommandName(namePtr); + while (token) { // Take this token and check to see if it's a group - if ((subGrp = group->FindSubGroupNoCase(ptr)) != nullptr) + if ((subGrp = group->FindSubGroupNoCase(*token)) != nullptr) group = subGrp; else break; - ptr = TokenizeCommandName(namePtr); + token = TokenizeCommandName(namePtr); } - if (ptr == nullptr) - { + if (!token) { fErrorMsg = ST_LITERAL("Invalid command syntax"); return {}; } /// OK, so what we found wasn't a group. Which means we need a command... - cmd = group->FindCommandNoCase( ptr ); + cmd = group->FindCommandNoCase(*token); if (cmd == nullptr) { fErrorMsg = ST_LITERAL("Invalid syntax: command not found"); @@ -300,35 +291,29 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S pfConsoleCmdGroup *group, *subGrp; int32_t numParams, i; pfConsoleCmdParam paramArray[ fMaxNumParams + 1 ]; - const char *ptr; bool valid = true; - - // TokenizeCommandName/TokenizeArguments requires a writable C string... - ST::char_buffer lineBuf = line.to_utf8(); - char* linePtr = lineBuf.data(); + const char* linePtr = line.c_str(); /// Loop #1: Scan for subgroups. This can be an empty loop group = pfConsoleCmdGroup::GetBaseGroup(); - ptr = TokenizeCommandName(linePtr); - while (ptr != nullptr) - { + auto token = TokenizeCommandName(linePtr); + while (token) { // Take this token and check to see if it's a group - if ((subGrp = group->FindSubGroupNoCase(ptr)) != nullptr) + if ((subGrp = group->FindSubGroupNoCase(*token)) != nullptr) group = subGrp; else break; - ptr = TokenizeCommandName(linePtr); + token = TokenizeCommandName(linePtr); } - if (ptr == nullptr) - { + if (!token) { fErrorMsg = ST_LITERAL("Invalid command syntax"); return false; } /// OK, so what we found wasn't a group. Which means we need a command next - cmd = group->FindCommandNoCase( ptr ); + cmd = group->FindCommandNoCase(*token); if (cmd == nullptr) { fErrorMsg = ST_LITERAL("Invalid syntax: command not found"); @@ -340,11 +325,10 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S /// params for (numParams = 0; numParams < fMaxNumParams - && (ptr = TokenizeArguments(linePtr)) != nullptr + && (token = TokenizeArguments(linePtr)) && valid; numParams++ ) { - if( ptr[ 0 ] == '\xFF' ) - { + if (*token == kTokenizeError) { fErrorMsg = ST_LITERAL("Invalid syntax: unterminated quoted parameter"); return false; } @@ -352,12 +336,11 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S // Special case for context variables--if we're specifying one, we want to just grab // the value of it and return that instead valid = false; - if( ptr[ 0 ] == '$' ) - { + if (token->starts_with("$")) { pfConsoleContext &context = pfConsoleContext::GetRootContext(); // Potential variable, see if we can find it - hsSsize_t idx = context.FindVar( ptr + 1 ); + hsSsize_t idx = context.FindVar(token->substr(1)); if( idx == -1 ) { fErrorMsg = ST_LITERAL("Invalid console variable name"); @@ -370,7 +353,7 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S } if( !valid ) - valid = IConvertToParam(cmd->GetSigEntry(numParams), ptr, ¶mArray[numParams]); + valid = IConvertToParam(cmd->GetSigEntry(numParams), *token, ¶mArray[numParams]); } for( i = numParams; i < fMaxNumParams + 1; i++ ) paramArray[ i ].SetNone(); @@ -461,31 +444,27 @@ ST::string pfConsoleEngine::FindPartialCmd(const ST::string& line, bool findAgai /// New search ST::string_stream newStr; - // TokenizeCommandName requires a writable C string... - ST::char_buffer lineBuf = line.to_utf8(); - char* linePtr = lineBuf.data(); + const char* linePtr = line.c_str(); /// Loop #1: Scan for subgroups. This can be an empty loop pfConsoleCmdGroup* group = pfConsoleCmdGroup::GetBaseGroup(); - const char* ptr = TokenizeCommandName(linePtr); - while (ptr != nullptr) - { + auto token = TokenizeCommandName(linePtr); + while (token) { // Take this token and check to see if it's a group - pfConsoleCmdGroup* subGrp = group->FindSubGroupNoCase(ptr, 0, nullptr); + pfConsoleCmdGroup* subGrp = group->FindSubGroupNoCase(*token, 0, nullptr); if (subGrp == nullptr) { break; } group = subGrp; newStr << group->GetName() << '.'; - ptr = TokenizeCommandName(linePtr); + token = TokenizeCommandName(linePtr); } - if (ptr != nullptr) - { + if (token) { // Still got at least one token left. Try to match to either // a partial group or a partial command - pfConsoleCmdGroup* subGrp = group->FindSubGroupNoCase(ptr, pfConsoleCmdGroup::kFindPartial, lastGroup); + pfConsoleCmdGroup* subGrp = group->FindSubGroupNoCase(*token, pfConsoleCmdGroup::kFindPartial, lastGroup); if (subGrp != nullptr) { lastGroup = group = subGrp; @@ -493,7 +472,7 @@ ST::string pfConsoleEngine::FindPartialCmd(const ST::string& line, bool findAgai } else { - pfConsoleCmd* cmd = group->FindCommandNoCase(ptr, pfConsoleCmdGroup::kFindPartial, lastCmd); + pfConsoleCmd* cmd = group->FindCommandNoCase(*token, pfConsoleCmdGroup::kFindPartial, lastCmd); if (cmd == nullptr) return {}; diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h index ab75a81bf5..8325036021 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h @@ -57,6 +57,7 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include "HeadSpin.h" +#include #include class plFileName; @@ -81,8 +82,18 @@ class pfConsoleEngine pfConsoleEngine(); ~pfConsoleEngine(); - static const char* TokenizeCommandName(char*& line); - static const char* TokenizeArguments(char*& line); + // Special value returned by the Tokenize methods (currently only TokenizeArguments) to indicate a syntax error. + static const ST::string kTokenizeError; + + // Parse the next command name or argument token from the given input line. + // On success, returns the parsed token. + // (Note that a successfully parsed token may be an empty string, e. g. from an empty pair of quotes!) + // The line pointer is incremented to point after that token + // so that it can be passed into another call to continue tokenizing. + // If the next token couldn't be parsed (e. g. quote not closed), returns kTokenizeError. + // If there are no further tokens in the line, returns an empty std::optional. + static std::optional TokenizeCommandName(const char*& line); + static std::optional TokenizeArguments(const char*& line); // Gets the signature for the command given (NO groups!) ST::string GetCmdSignature(const ST::string& name); diff --git a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp index 5967af9fa0..5dd534f999 100644 --- a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp +++ b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp @@ -44,33 +44,35 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include "pfConsoleCore/pfConsoleEngine.h" +using namespace ST::literals; + // Tokenize command name, 1 token TEST(pfConsoleCore, TokenizeCommandNameSingle) { - char buf[] = "SampleCmd1"; - char* line = buf; + const char buf[] = "SampleCmd1"; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_STREQ(token1, "SampleCmd1"); + auto token1 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_EQ(token1, "SampleCmd1"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token2 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_EQ(token2, nullptr); + auto token2 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_FALSE(token2); EXPECT_EQ(line, buf + sizeof(buf) - 1); } TEST(pfConsoleCore, TokenizeCommandNameSingleWhitespace) { - char buf[] = " SampleCmd1 "; - char* line = buf; + const char buf[] = " SampleCmd1 "; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_STREQ(token1, "SampleCmd1"); + auto token1 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_EQ(token1, "SampleCmd1"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token2 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_EQ(token2, nullptr); + auto token2 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_FALSE(token2); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -78,55 +80,55 @@ TEST(pfConsoleCore, TokenizeCommandNameSingleWhitespace) TEST(pfConsoleCore, TokenizeCommandNameDot) { - char buf[] = "App.Quit"; - char* line = buf; + const char buf[] = "App.Quit"; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_STREQ(token1, "App"); + auto token1 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_EQ(token1, "App"_st); EXPECT_EQ(line, buf + sizeof("App.") - 1); - const char* token2 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_STREQ(token2, "Quit"); + auto token2 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_EQ(token2, "Quit"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token3 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_EQ(token3, nullptr); + auto token3 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_FALSE(token3); EXPECT_EQ(line, buf + sizeof(buf) - 1); } TEST(pfConsoleCore, TokenizeCommandNameUnderscore) { - char buf[] = "App_Quit"; - char* line = buf; + const char buf[] = "App_Quit"; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_STREQ(token1, "App"); + auto token1 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_EQ(token1, "App"_st); EXPECT_EQ(line, buf + sizeof("App_") - 1); - const char* token2 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_STREQ(token2, "Quit"); + auto token2 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_EQ(token2, "Quit"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token3 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_EQ(token3, nullptr); + auto token3 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_FALSE(token3); EXPECT_EQ(line, buf + sizeof(buf) - 1); } TEST(pfConsoleCore, TokenizeCommandNameSpace) { - char buf[] = "App Quit"; - char* line = buf; + const char buf[] = "App Quit"; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_STREQ(token1, "App"); + auto token1 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_EQ(token1, "App"_st); EXPECT_EQ(line, buf + sizeof("App ") - 1); - const char* token2 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_STREQ(token2, "Quit"); + auto token2 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_EQ(token2, "Quit"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token3 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_EQ(token3, nullptr); + auto token3 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_FALSE(token3); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -134,67 +136,67 @@ TEST(pfConsoleCore, TokenizeCommandNameSpace) TEST(pfConsoleCore, TokenizeCommandNameDots) { - char buf[] = "Graphics.Renderer.SetYon"; - char* line = buf; + const char buf[] = "Graphics.Renderer.SetYon"; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_STREQ(token1, "Graphics"); + auto token1 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_EQ(token1, "Graphics"_st); EXPECT_EQ(line, buf + sizeof("Graphics.") - 1); - const char* token2 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_STREQ(token2, "Renderer"); + auto token2 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_EQ(token2, "Renderer"_st); EXPECT_EQ(line, buf + sizeof("Graphics.Renderer.") - 1); - const char* token3 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_STREQ(token3, "SetYon"); + auto token3 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_EQ(token3, "SetYon"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token4 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_EQ(token4, nullptr); + auto token4 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_FALSE(token4); EXPECT_EQ(line, buf + sizeof(buf) - 1); } TEST(pfConsoleCore, TokenizeCommandNameUnderscores) { - char buf[] = "Graphics_Renderer_SetYon"; - char* line = buf; + const char buf[] = "Graphics_Renderer_SetYon"; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_STREQ(token1, "Graphics"); + auto token1 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_EQ(token1, "Graphics"_st); EXPECT_EQ(line, buf + sizeof("Graphics_") - 1); - const char* token2 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_STREQ(token2, "Renderer"); + auto token2 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_EQ(token2, "Renderer"_st); EXPECT_EQ(line, buf + sizeof("Graphics_Renderer_") - 1); - const char* token3 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_STREQ(token3, "SetYon"); + auto token3 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_EQ(token3, "SetYon"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token4 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_EQ(token4, nullptr); + auto token4 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_FALSE(token4); EXPECT_EQ(line, buf + sizeof(buf) - 1); } TEST(pfConsoleCore, TokenizeCommandNameSpaces) { - char buf[] = "Graphics Renderer SetYon"; - char* line = buf; + const char buf[] = "Graphics Renderer SetYon"; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_STREQ(token1, "Graphics"); + auto token1 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_EQ(token1, "Graphics"_st); EXPECT_EQ(line, buf + sizeof("Graphics ") - 1); - const char* token2 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_STREQ(token2, "Renderer"); + auto token2 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_EQ(token2, "Renderer"_st); EXPECT_EQ(line, buf + sizeof("Graphics Renderer ") - 1); - const char* token3 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_STREQ(token3, "SetYon"); + auto token3 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_EQ(token3, "SetYon"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token4 = pfConsoleEngine::TokenizeCommandName(line); - EXPECT_EQ(token4, nullptr); + auto token4 = pfConsoleEngine::TokenizeCommandName(line); + EXPECT_FALSE(token4); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -202,163 +204,163 @@ TEST(pfConsoleCore, TokenizeCommandNameSpaces) TEST(pfConsoleCore, TokenizeArgumentsSingle) { - char buf[] = "arg"; - char* line = buf; + const char buf[] = "arg"; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token1, "arg"); + auto token1 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token1, "arg"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token2 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_EQ(token2, nullptr); + auto token2 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_FALSE(token2); EXPECT_EQ(line, buf + sizeof(buf) - 1); } TEST(pfConsoleCore, TokenizeArgumentsSingleWhitespace) { - char buf[] = " arg "; - char* line = buf; + const char buf[] = " arg "; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token1, "arg"); + auto token1 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token1, "arg"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token2 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_EQ(token2, nullptr); + auto token2 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_FALSE(token2); EXPECT_EQ(line, buf + sizeof(buf) - 1); } TEST(pfConsoleCore, TokenizeArgumentsSingleUnderscore) { - char buf[] = "arg_test"; - char* line = buf; + const char buf[] = "arg_test"; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token1, "arg_test"); + auto token1 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token1, "arg_test"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token2 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_EQ(token2, nullptr); + auto token2 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_FALSE(token2); EXPECT_EQ(line, buf + sizeof(buf) - 1); } TEST(pfConsoleCore, TokenizeArgumentsSingleDoubleQuote) { - char buf[] = "\"(Default Device)\""; - char* line = buf; + const char buf[] = "\"(Default Device)\""; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token1, "(Default Device)"); + auto token1 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token1, "(Default Device)"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token2 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_EQ(token2, nullptr); + auto token2 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_FALSE(token2); EXPECT_EQ(line, buf + sizeof(buf) - 1); } TEST(pfConsoleCore, TokenizeArgumentsSingleSingleQuote) { - char buf[] = "'(Default Device)'"; - char* line = buf; + const char buf[] = "'(Default Device)'"; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token1, "(Default Device)"); + auto token1 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token1, "(Default Device)"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token2 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_EQ(token2, nullptr); + auto token2 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_FALSE(token2); EXPECT_EQ(line, buf + sizeof(buf) - 1); } TEST(pfConsoleCore, TokenizeArgumentsSingleDoubleQuoteUnclosed) { - char buf[] = "\"(Default Device)"; - char* line = buf; + const char buf[] = "\"(Default Device)"; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token1, "\xff"); + auto token1 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token1, "\xff"_st); } TEST(pfConsoleCore, TokenizeArgumentsSingleSingleQuoteUnclosed) { - char buf[] = "'(Default Device)"; - char* line = buf; + const char buf[] = "'(Default Device)"; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token1, "\xff"); + auto token1 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token1, "\xff"_st); } // Tokenize arguments, 2 tokens TEST(pfConsoleCore, TokenizeArgumentsPair) { - char buf[] = "arg1 arg2"; - char* line = buf; + const char buf[] = "arg1 arg2"; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token1, "arg1"); + auto token1 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token1, "arg1"_st); EXPECT_EQ(line, buf + sizeof("arg1 ") - 1); - const char* token2 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token2, "arg2"); + auto token2 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token2, "arg2"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token3 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_EQ(token3, nullptr); + auto token3 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_FALSE(token3); EXPECT_EQ(line, buf + sizeof(buf) - 1); } TEST(pfConsoleCore, TokenizeArgumentsPairWhitespace) { - char buf[] = " arg1 arg2 "; - char* line = buf; + const char buf[] = " arg1 arg2 "; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token1, "arg1"); + auto token1 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token1, "arg1"_st); EXPECT_EQ(line, buf + sizeof(" arg1 ") - 1); - const char* token2 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token2, "arg2"); + auto token2 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token2, "arg2"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token3 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_EQ(token3, nullptr); + auto token3 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_FALSE(token3); EXPECT_EQ(line, buf + sizeof(buf) - 1); } TEST(pfConsoleCore, TokenizeArgumentsPairComma) { - char buf[] = "arg1, arg2"; - char* line = buf; + const char buf[] = "arg1, arg2"; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token1, "arg1"); + auto token1 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token1, "arg1"_st); EXPECT_EQ(line, buf + sizeof("arg1,") - 1); - const char* token2 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token2, "arg2"); + auto token2 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token2, "arg2"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token3 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_EQ(token3, nullptr); + auto token3 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_FALSE(token3); EXPECT_EQ(line, buf + sizeof(buf) - 1); } TEST(pfConsoleCore, TokenizeArgumentsPairMixedQuotes) { - char buf[] = "\"argument '1'\" 'argument \"2\"'"; - char* line = buf; + const char buf[] = "\"argument '1'\" 'argument \"2\"'"; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token1, "argument '1'"); + auto token1 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token1, "argument '1'"_st); EXPECT_EQ(line, buf + sizeof("\"argument '1'\"") - 1); - const char* token2 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token2, "argument \"2\""); + auto token2 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token2, "argument \"2\""_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token3 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_EQ(token3, nullptr); + auto token3 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_FALSE(token3); EXPECT_EQ(line, buf + sizeof(buf) - 1); } @@ -366,44 +368,44 @@ TEST(pfConsoleCore, TokenizeArgumentsPairMixedQuotes) TEST(pfConsoleCore, TokenizeArgumentsTriple) { - char buf[] = "1.2 3.4 5.6"; - char* line = buf; + const char buf[] = "1.2 3.4 5.6"; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token1, "1.2"); + auto token1 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token1, "1.2"_st); EXPECT_EQ(line, buf + sizeof("1.2 ") - 1); - const char* token2 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token2, "3.4"); + auto token2 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token2, "3.4"_st); EXPECT_EQ(line, buf + sizeof("1.2 3.4 ") - 1); - const char* token3 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token3, "5.6"); + auto token3 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token3, "5.6"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token4 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_EQ(token4, nullptr); + auto token4 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_FALSE(token4); EXPECT_EQ(line, buf + sizeof(buf) - 1); } TEST(pfConsoleCore, TokenizeArgumentsTripleCommas) { - char buf[] = "1.2, 3.4, 5.6"; - char* line = buf; + const char buf[] = "1.2, 3.4, 5.6"; + const char* line = buf; - const char* token1 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token1, "1.2"); + auto token1 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token1, "1.2"_st); EXPECT_EQ(line, buf + sizeof("1.2,") - 1); - const char* token2 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token2, "3.4"); + auto token2 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token2, "3.4"_st); EXPECT_EQ(line, buf + sizeof("1.2, 3.4,") - 1); - const char* token3 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_STREQ(token3, "5.6"); + auto token3 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token3, "5.6"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - const char* token4 = pfConsoleEngine::TokenizeArguments(line); - EXPECT_EQ(token4, nullptr); + auto token4 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_FALSE(token4); EXPECT_EQ(line, buf + sizeof(buf) - 1); } From e28d0739dd1dc8b38e5c85c636a933510d46dc2a Mon Sep 17 00:00:00 2001 From: dgelessus Date: Sat, 29 Jul 2023 15:34:00 +0200 Subject: [PATCH 06/17] Add pfConsoleEngine tokenization unit test for empty quotes --- .../pfConsoleCoreTest/test_pfConsoleCore.cpp | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp index 5dd534f999..3e27fda937 100644 --- a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp +++ b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp @@ -409,3 +409,25 @@ TEST(pfConsoleCore, TokenizeArgumentsTripleCommas) EXPECT_FALSE(token4); EXPECT_EQ(line, buf + sizeof(buf) - 1); } + +TEST(pfConsoleCore, TokenizeArgumentsTripleEmptyQuotes) +{ + const char buf[] = "'' \"\" ''"; + const char* line = buf; + + auto token1 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token1, ""_st); + EXPECT_EQ(line, buf + sizeof("''") - 1); + + auto token2 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token2, ""_st); + EXPECT_EQ(line, buf + sizeof("'' \"\"") - 1); + + auto token3 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_EQ(token3, ""_st); + EXPECT_EQ(line, buf + sizeof(buf) - 1); + + auto token4 = pfConsoleEngine::TokenizeArguments(line); + EXPECT_FALSE(token4); + EXPECT_EQ(line, buf + sizeof(buf) - 1); +} From cc1b84bbb95716faca1c94d6f09ff6e1f2ac137d Mon Sep 17 00:00:00 2001 From: dgelessus Date: Sun, 30 Jul 2023 02:42:52 +0200 Subject: [PATCH 07/17] Split pfConsole tokenization code into its own class In preparation for refactoring and extending it into a more usable standalone parser. --- .../FeatureLib/pfConsoleCore/CMakeLists.txt | 2 + .../pfConsoleCore/pfConsoleEngine.cpp | 88 ++------- .../pfConsoleCore/pfConsoleEngine.h | 14 -- .../pfConsoleCore/pfConsoleParser.cpp | 113 ++++++++++++ .../pfConsoleCore/pfConsoleParser.h | 69 +++++++ .../pfConsoleCoreTest/CMakeLists.txt | 2 +- ...leCore.cpp => test_pfConsoleTokenizer.cpp} | 169 +++++++++--------- 7 files changed, 281 insertions(+), 176 deletions(-) create mode 100644 Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp create mode 100644 Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h rename Sources/Tests/FeatureTests/pfConsoleCoreTest/{test_pfConsoleCore.cpp => test_pfConsoleTokenizer.cpp} (61%) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/CMakeLists.txt b/Sources/Plasma/FeatureLib/pfConsoleCore/CMakeLists.txt index 4b318c8a57..ea3bab9e24 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/CMakeLists.txt +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/CMakeLists.txt @@ -3,12 +3,14 @@ set(pfConsoleCore_SOURCES pfConsoleCommandsCore.cpp pfConsoleContext.cpp pfConsoleEngine.cpp + pfConsoleParser.cpp ) set(pfConsoleCore_HEADERS pfConsoleCmd.h pfConsoleContext.h pfConsoleEngine.h + pfConsoleParser.h ) plasma_library(pfConsoleCore SOURCES ${pfConsoleCore_SOURCES} ${pfConsoleCore_HEADERS}) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp index 6fff6dfa12..b33f64327e 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp @@ -48,6 +48,7 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include "pfConsoleEngine.h" #include "pfConsoleCmd.h" #include "pfConsoleContext.h" +#include "pfConsoleParser.h" #include #include @@ -58,73 +59,6 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com const int32_t pfConsoleEngine::fMaxNumParams = 16; -// Yes, this is invalid UTF-8. Yes, this is awful. -const ST::string pfConsoleEngine::kTokenizeError = ST_LITERAL("\xff"); - -static const char kTokenSeparators[] = " =\r\n\t,"; -static const char kTokenGrpSeps[] = " =\r\n._\t,"; - -//WARNING: Potentially increments the pointer passed to it. -std::optional pfConsoleEngine::TokenizeCommandName(const char*& line) -{ - const char* begin = line; - - while (*begin && isspace(static_cast(*begin))) - ++begin; - - for (line = begin; *line; ++line) { - for (const char *sep = kTokenGrpSeps; *sep; ++sep) { - if (*line == *sep) { - const char* end = line; - while (*++line && (*line == *sep)) - /* skip duplicate delimiters */; - return ST::string::from_utf8(begin, end - begin); - } - } - } - - if (begin == line) { - return {}; - } - - return begin; -} - -//WARNING: Potentially increments the pointer passed to it. -std::optional pfConsoleEngine::TokenizeArguments(const char*& line) -{ - const char* begin = line; - - while (*begin && isspace(static_cast(*begin))) - ++begin; - - for (line = begin; *line; ++line) { - if (*begin == '"' || *begin == '\'') { - // Handle strings as a single token - ++begin; - const char* end = strchr(begin, *line); - if (end == nullptr) { - return kTokenizeError; - } - line = end + 1; - return ST::string::from_utf8(begin, end - begin); - } - for (const char *sep = kTokenSeparators; *sep; ++sep) { - if (*line == *sep) { - const char* end = line; - while (*++line && (*line == *sep)) - /* skip duplicate delimiters */; - return ST::string::from_utf8(begin, end - begin); - } - } - } - - if (begin == line) { - return {}; - } - - return begin; -} //// Constructor & Destructor //////////////////////////////////////////////// @@ -147,7 +81,7 @@ bool pfConsoleEngine::PrintCmdHelp(const ST::string& name, void (*PrintFn)(const /// Scan for subgroups. This can be an empty loop group = pfConsoleCmdGroup::GetBaseGroup(); - auto token = TokenizeCommandName(namePtr); + auto token = pfConsoleTokenizer::TokenizeCommandName(namePtr); while (token) { // Take this token and check to see if it's a group if ((subGrp = group->FindSubGroupNoCase(*token)) != nullptr) @@ -155,7 +89,7 @@ bool pfConsoleEngine::PrintCmdHelp(const ST::string& name, void (*PrintFn)(const else break; - token = TokenizeCommandName(namePtr); + token = pfConsoleTokenizer::TokenizeCommandName(namePtr); } if (!token) { @@ -210,7 +144,7 @@ ST::string pfConsoleEngine::GetCmdSignature(const ST::string& name) /// Scan for subgroups. This can be an empty loop group = pfConsoleCmdGroup::GetBaseGroup(); - auto token = TokenizeCommandName(namePtr); + auto token = pfConsoleTokenizer::TokenizeCommandName(namePtr); while (token) { // Take this token and check to see if it's a group if ((subGrp = group->FindSubGroupNoCase(*token)) != nullptr) @@ -218,7 +152,7 @@ ST::string pfConsoleEngine::GetCmdSignature(const ST::string& name) else break; - token = TokenizeCommandName(namePtr); + token = pfConsoleTokenizer::TokenizeCommandName(namePtr); } if (!token) { @@ -296,7 +230,7 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S /// Loop #1: Scan for subgroups. This can be an empty loop group = pfConsoleCmdGroup::GetBaseGroup(); - auto token = TokenizeCommandName(linePtr); + auto token = pfConsoleTokenizer::TokenizeCommandName(linePtr); while (token) { // Take this token and check to see if it's a group if ((subGrp = group->FindSubGroupNoCase(*token)) != nullptr) @@ -304,7 +238,7 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S else break; - token = TokenizeCommandName(linePtr); + token = pfConsoleTokenizer::TokenizeCommandName(linePtr); } if (!token) { @@ -325,10 +259,10 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S /// params for (numParams = 0; numParams < fMaxNumParams - && (token = TokenizeArguments(linePtr)) + && (token = pfConsoleTokenizer::TokenizeArguments(linePtr)) && valid; numParams++ ) { - if (*token == kTokenizeError) { + if (*token == pfConsoleTokenizer::kTokenizeError) { fErrorMsg = ST_LITERAL("Invalid syntax: unterminated quoted parameter"); return false; } @@ -448,7 +382,7 @@ ST::string pfConsoleEngine::FindPartialCmd(const ST::string& line, bool findAgai /// Loop #1: Scan for subgroups. This can be an empty loop pfConsoleCmdGroup* group = pfConsoleCmdGroup::GetBaseGroup(); - auto token = TokenizeCommandName(linePtr); + auto token = pfConsoleTokenizer::TokenizeCommandName(linePtr); while (token) { // Take this token and check to see if it's a group pfConsoleCmdGroup* subGrp = group->FindSubGroupNoCase(*token, 0, nullptr); @@ -458,7 +392,7 @@ ST::string pfConsoleEngine::FindPartialCmd(const ST::string& line, bool findAgai group = subGrp; newStr << group->GetName() << '.'; - token = TokenizeCommandName(linePtr); + token = pfConsoleTokenizer::TokenizeCommandName(linePtr); } if (token) { diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h index 8325036021..0b3c25421b 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h @@ -57,7 +57,6 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include "HeadSpin.h" -#include #include class plFileName; @@ -82,19 +81,6 @@ class pfConsoleEngine pfConsoleEngine(); ~pfConsoleEngine(); - // Special value returned by the Tokenize methods (currently only TokenizeArguments) to indicate a syntax error. - static const ST::string kTokenizeError; - - // Parse the next command name or argument token from the given input line. - // On success, returns the parsed token. - // (Note that a successfully parsed token may be an empty string, e. g. from an empty pair of quotes!) - // The line pointer is incremented to point after that token - // so that it can be passed into another call to continue tokenizing. - // If the next token couldn't be parsed (e. g. quote not closed), returns kTokenizeError. - // If there are no further tokens in the line, returns an empty std::optional. - static std::optional TokenizeCommandName(const char*& line); - static std::optional TokenizeArguments(const char*& line); - // Gets the signature for the command given (NO groups!) ST::string GetCmdSignature(const ST::string& name); diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp new file mode 100644 index 0000000000..98d60b6aec --- /dev/null +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp @@ -0,0 +1,113 @@ +/*==LICENSE==* + +CyanWorlds.com Engine - MMOG client, server and tools +Copyright (C) 2011 Cyan Worlds, Inc. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Additional permissions under GNU GPL version 3 section 7 + +If you modify this Program, or any covered work, by linking or +combining it with any of RAD Game Tools Bink SDK, Autodesk 3ds Max SDK, +NVIDIA PhysX SDK, Microsoft DirectX SDK, OpenSSL library, Independent +JPEG Group JPEG library, Microsoft Windows Media SDK, or Apple QuickTime SDK +(or a modified version of those libraries), +containing parts covered by the terms of the Bink SDK EULA, 3ds Max EULA, +PhysX SDK EULA, DirectX SDK EULA, OpenSSL and SSLeay licenses, IJG +JPEG Library README, Windows Media SDK EULA, or QuickTime SDK EULA, the +licensors of this Program grant you additional +permission to convey the resulting work. Corresponding Source for a +non-source form of such a combination shall include the source code for +the parts of OpenSSL and IJG JPEG Library used as well as that of the covered +work. + +You can contact Cyan Worlds, Inc. by email legal@cyan.com + or by snail mail at: + Cyan Worlds, Inc. + 14617 N Newport Hwy + Mead, WA 99021 + +*==LICENSE==*/ + +#include "pfConsoleParser.h" + +#include + +// Yes, this is invalid UTF-8. Yes, this is awful. +const ST::string pfConsoleTokenizer::kTokenizeError = ST_LITERAL("\xff"); + +static const char kTokenSeparators[] = " =\r\n\t,"; +static const char kTokenGrpSeps[] = " =\r\n._\t,"; + +//WARNING: Potentially increments the pointer passed to it. +std::optional pfConsoleTokenizer::TokenizeCommandName(const char*& line) +{ + const char* begin = line; + + while (*begin && isspace(static_cast(*begin))) + ++begin; + + for (line = begin; *line; ++line) { + for (const char *sep = kTokenGrpSeps; *sep; ++sep) { + if (*line == *sep) { + const char* end = line; + while (*++line && (*line == *sep)) + /* skip duplicate delimiters */; + return ST::string::from_utf8(begin, end - begin); + } + } + } + + if (begin == line) { + return {}; + } + + return begin; +} + +//WARNING: Potentially increments the pointer passed to it. +std::optional pfConsoleTokenizer::TokenizeArguments(const char*& line) +{ + const char* begin = line; + + while (*begin && isspace(static_cast(*begin))) + ++begin; + + for (line = begin; *line; ++line) { + if (*begin == '"' || *begin == '\'') { + // Handle strings as a single token + ++begin; + const char* end = strchr(begin, *line); + if (end == nullptr) { + return kTokenizeError; + } + line = end + 1; + return ST::string::from_utf8(begin, end - begin); + } + for (const char *sep = kTokenSeparators; *sep; ++sep) { + if (*line == *sep) { + const char* end = line; + while (*++line && (*line == *sep)) + /* skip duplicate delimiters */; + return ST::string::from_utf8(begin, end - begin); + } + } + } + + if (begin == line) { + return {}; + } + + return begin; +} diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h new file mode 100644 index 0000000000..675e325dff --- /dev/null +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h @@ -0,0 +1,69 @@ +/*==LICENSE==* + +CyanWorlds.com Engine - MMOG client, server and tools +Copyright (C) 2011 Cyan Worlds, Inc. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Additional permissions under GNU GPL version 3 section 7 + +If you modify this Program, or any covered work, by linking or +combining it with any of RAD Game Tools Bink SDK, Autodesk 3ds Max SDK, +NVIDIA PhysX SDK, Microsoft DirectX SDK, OpenSSL library, Independent +JPEG Group JPEG library, Microsoft Windows Media SDK, or Apple QuickTime SDK +(or a modified version of those libraries), +containing parts covered by the terms of the Bink SDK EULA, 3ds Max EULA, +PhysX SDK EULA, DirectX SDK EULA, OpenSSL and SSLeay licenses, IJG +JPEG Library README, Windows Media SDK EULA, or QuickTime SDK EULA, the +licensors of this Program grant you additional +permission to convey the resulting work. Corresponding Source for a +non-source form of such a combination shall include the source code for +the parts of OpenSSL and IJG JPEG Library used as well as that of the covered +work. + +You can contact Cyan Worlds, Inc. by email legal@cyan.com + or by snail mail at: + Cyan Worlds, Inc. + 14617 N Newport Hwy + Mead, WA 99021 + +*==LICENSE==*/ + +#ifndef _pfConsolePareser_h +#define _pfConsolePareser_h + +#include "HeadSpin.h" + +#include + +namespace ST { class string; } + +class pfConsoleTokenizer +{ +public: + // Special value returned by the Tokenize methods (currently only TokenizeArguments) to indicate a syntax error. + static const ST::string kTokenizeError; + + // Parse the next command name or argument token from the given input line. + // On success, returns the parsed token. + // (Note that a successfully parsed token may be an empty string, e. g. from an empty pair of quotes!) + // The line pointer is incremented to point after that token + // so that it can be passed into another call to continue tokenizing. + // If the next token couldn't be parsed (e. g. quote not closed), returns kTokenizeError. + // If there are no further tokens in the line, returns an empty std::optional. + static std::optional TokenizeCommandName(const char*& line); + static std::optional TokenizeArguments(const char*& line); +}; + +#endif // _pfConsolePareser_h diff --git a/Sources/Tests/FeatureTests/pfConsoleCoreTest/CMakeLists.txt b/Sources/Tests/FeatureTests/pfConsoleCoreTest/CMakeLists.txt index 24d389a189..4676bb3871 100644 --- a/Sources/Tests/FeatureTests/pfConsoleCoreTest/CMakeLists.txt +++ b/Sources/Tests/FeatureTests/pfConsoleCoreTest/CMakeLists.txt @@ -1,5 +1,5 @@ set(pfConsoleCore_SOURCES - test_pfConsoleCore.cpp + test_pfConsoleTokenizer.cpp ) plasma_test(test_pfConsoleCore SOURCES ${pfConsoleCore_SOURCES}) diff --git a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleTokenizer.cpp similarity index 61% rename from Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp rename to Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleTokenizer.cpp index 3e27fda937..dd86783a61 100644 --- a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleCore.cpp +++ b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleTokenizer.cpp @@ -41,393 +41,394 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com *==LICENSE==*/ #include +#include -#include "pfConsoleCore/pfConsoleEngine.h" +#include "pfConsoleCore/pfConsoleParser.h" using namespace ST::literals; // Tokenize command name, 1 token -TEST(pfConsoleCore, TokenizeCommandNameSingle) +TEST(pfConsoleTokenizer, TokenizeCommandNameSingle) { const char buf[] = "SampleCmd1"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeCommandName(line); + auto token1 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_EQ(token1, "SampleCmd1"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token2 = pfConsoleEngine::TokenizeCommandName(line); + auto token2 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_FALSE(token2); EXPECT_EQ(line, buf + sizeof(buf) - 1); } -TEST(pfConsoleCore, TokenizeCommandNameSingleWhitespace) +TEST(pfConsoleTokenizer, TokenizeCommandNameSingleWhitespace) { const char buf[] = " SampleCmd1 "; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeCommandName(line); + auto token1 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_EQ(token1, "SampleCmd1"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token2 = pfConsoleEngine::TokenizeCommandName(line); + auto token2 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_FALSE(token2); EXPECT_EQ(line, buf + sizeof(buf) - 1); } // Tokenize command name, 2 tokens -TEST(pfConsoleCore, TokenizeCommandNameDot) +TEST(pfConsoleTokenizer, TokenizeCommandNameDot) { const char buf[] = "App.Quit"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeCommandName(line); + auto token1 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_EQ(token1, "App"_st); EXPECT_EQ(line, buf + sizeof("App.") - 1); - auto token2 = pfConsoleEngine::TokenizeCommandName(line); + auto token2 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_EQ(token2, "Quit"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token3 = pfConsoleEngine::TokenizeCommandName(line); + auto token3 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_FALSE(token3); EXPECT_EQ(line, buf + sizeof(buf) - 1); } -TEST(pfConsoleCore, TokenizeCommandNameUnderscore) +TEST(pfConsoleTokenizer, TokenizeCommandNameUnderscore) { const char buf[] = "App_Quit"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeCommandName(line); + auto token1 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_EQ(token1, "App"_st); EXPECT_EQ(line, buf + sizeof("App_") - 1); - auto token2 = pfConsoleEngine::TokenizeCommandName(line); + auto token2 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_EQ(token2, "Quit"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token3 = pfConsoleEngine::TokenizeCommandName(line); + auto token3 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_FALSE(token3); EXPECT_EQ(line, buf + sizeof(buf) - 1); } -TEST(pfConsoleCore, TokenizeCommandNameSpace) +TEST(pfConsoleTokenizer, TokenizeCommandNameSpace) { const char buf[] = "App Quit"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeCommandName(line); + auto token1 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_EQ(token1, "App"_st); EXPECT_EQ(line, buf + sizeof("App ") - 1); - auto token2 = pfConsoleEngine::TokenizeCommandName(line); + auto token2 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_EQ(token2, "Quit"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token3 = pfConsoleEngine::TokenizeCommandName(line); + auto token3 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_FALSE(token3); EXPECT_EQ(line, buf + sizeof(buf) - 1); } // Tokenize command name, 3 tokens -TEST(pfConsoleCore, TokenizeCommandNameDots) +TEST(pfConsoleTokenizer, TokenizeCommandNameDots) { const char buf[] = "Graphics.Renderer.SetYon"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeCommandName(line); + auto token1 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_EQ(token1, "Graphics"_st); EXPECT_EQ(line, buf + sizeof("Graphics.") - 1); - auto token2 = pfConsoleEngine::TokenizeCommandName(line); + auto token2 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_EQ(token2, "Renderer"_st); EXPECT_EQ(line, buf + sizeof("Graphics.Renderer.") - 1); - auto token3 = pfConsoleEngine::TokenizeCommandName(line); + auto token3 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_EQ(token3, "SetYon"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token4 = pfConsoleEngine::TokenizeCommandName(line); + auto token4 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_FALSE(token4); EXPECT_EQ(line, buf + sizeof(buf) - 1); } -TEST(pfConsoleCore, TokenizeCommandNameUnderscores) +TEST(pfConsoleTokenizer, TokenizeCommandNameUnderscores) { const char buf[] = "Graphics_Renderer_SetYon"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeCommandName(line); + auto token1 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_EQ(token1, "Graphics"_st); EXPECT_EQ(line, buf + sizeof("Graphics_") - 1); - auto token2 = pfConsoleEngine::TokenizeCommandName(line); + auto token2 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_EQ(token2, "Renderer"_st); EXPECT_EQ(line, buf + sizeof("Graphics_Renderer_") - 1); - auto token3 = pfConsoleEngine::TokenizeCommandName(line); + auto token3 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_EQ(token3, "SetYon"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token4 = pfConsoleEngine::TokenizeCommandName(line); + auto token4 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_FALSE(token4); EXPECT_EQ(line, buf + sizeof(buf) - 1); } -TEST(pfConsoleCore, TokenizeCommandNameSpaces) +TEST(pfConsoleTokenizer, TokenizeCommandNameSpaces) { const char buf[] = "Graphics Renderer SetYon"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeCommandName(line); + auto token1 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_EQ(token1, "Graphics"_st); EXPECT_EQ(line, buf + sizeof("Graphics ") - 1); - auto token2 = pfConsoleEngine::TokenizeCommandName(line); + auto token2 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_EQ(token2, "Renderer"_st); EXPECT_EQ(line, buf + sizeof("Graphics Renderer ") - 1); - auto token3 = pfConsoleEngine::TokenizeCommandName(line); + auto token3 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_EQ(token3, "SetYon"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token4 = pfConsoleEngine::TokenizeCommandName(line); + auto token4 = pfConsoleTokenizer::TokenizeCommandName(line); EXPECT_FALSE(token4); EXPECT_EQ(line, buf + sizeof(buf) - 1); } // Tokenize arguments, 1 token -TEST(pfConsoleCore, TokenizeArgumentsSingle) +TEST(pfConsoleTokenizer, TokenizeArgumentsSingle) { const char buf[] = "arg"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeArguments(line); + auto token1 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token1, "arg"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token2 = pfConsoleEngine::TokenizeArguments(line); + auto token2 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_FALSE(token2); EXPECT_EQ(line, buf + sizeof(buf) - 1); } -TEST(pfConsoleCore, TokenizeArgumentsSingleWhitespace) +TEST(pfConsoleTokenizer, TokenizeArgumentsSingleWhitespace) { const char buf[] = " arg "; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeArguments(line); + auto token1 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token1, "arg"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token2 = pfConsoleEngine::TokenizeArguments(line); + auto token2 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_FALSE(token2); EXPECT_EQ(line, buf + sizeof(buf) - 1); } -TEST(pfConsoleCore, TokenizeArgumentsSingleUnderscore) +TEST(pfConsoleTokenizer, TokenizeArgumentsSingleUnderscore) { const char buf[] = "arg_test"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeArguments(line); + auto token1 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token1, "arg_test"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token2 = pfConsoleEngine::TokenizeArguments(line); + auto token2 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_FALSE(token2); EXPECT_EQ(line, buf + sizeof(buf) - 1); } -TEST(pfConsoleCore, TokenizeArgumentsSingleDoubleQuote) +TEST(pfConsoleTokenizer, TokenizeArgumentsSingleDoubleQuote) { const char buf[] = "\"(Default Device)\""; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeArguments(line); + auto token1 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token1, "(Default Device)"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token2 = pfConsoleEngine::TokenizeArguments(line); + auto token2 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_FALSE(token2); EXPECT_EQ(line, buf + sizeof(buf) - 1); } -TEST(pfConsoleCore, TokenizeArgumentsSingleSingleQuote) +TEST(pfConsoleTokenizer, TokenizeArgumentsSingleSingleQuote) { const char buf[] = "'(Default Device)'"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeArguments(line); + auto token1 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token1, "(Default Device)"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token2 = pfConsoleEngine::TokenizeArguments(line); + auto token2 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_FALSE(token2); EXPECT_EQ(line, buf + sizeof(buf) - 1); } -TEST(pfConsoleCore, TokenizeArgumentsSingleDoubleQuoteUnclosed) +TEST(pfConsoleTokenizer, TokenizeArgumentsSingleDoubleQuoteUnclosed) { const char buf[] = "\"(Default Device)"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeArguments(line); + auto token1 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token1, "\xff"_st); } -TEST(pfConsoleCore, TokenizeArgumentsSingleSingleQuoteUnclosed) +TEST(pfConsoleTokenizer, TokenizeArgumentsSingleSingleQuoteUnclosed) { const char buf[] = "'(Default Device)"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeArguments(line); + auto token1 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token1, "\xff"_st); } // Tokenize arguments, 2 tokens -TEST(pfConsoleCore, TokenizeArgumentsPair) +TEST(pfConsoleTokenizer, TokenizeArgumentsPair) { const char buf[] = "arg1 arg2"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeArguments(line); + auto token1 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token1, "arg1"_st); EXPECT_EQ(line, buf + sizeof("arg1 ") - 1); - auto token2 = pfConsoleEngine::TokenizeArguments(line); + auto token2 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token2, "arg2"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token3 = pfConsoleEngine::TokenizeArguments(line); + auto token3 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_FALSE(token3); EXPECT_EQ(line, buf + sizeof(buf) - 1); } -TEST(pfConsoleCore, TokenizeArgumentsPairWhitespace) +TEST(pfConsoleTokenizer, TokenizeArgumentsPairWhitespace) { const char buf[] = " arg1 arg2 "; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeArguments(line); + auto token1 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token1, "arg1"_st); EXPECT_EQ(line, buf + sizeof(" arg1 ") - 1); - auto token2 = pfConsoleEngine::TokenizeArguments(line); + auto token2 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token2, "arg2"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token3 = pfConsoleEngine::TokenizeArguments(line); + auto token3 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_FALSE(token3); EXPECT_EQ(line, buf + sizeof(buf) - 1); } -TEST(pfConsoleCore, TokenizeArgumentsPairComma) +TEST(pfConsoleTokenizer, TokenizeArgumentsPairComma) { const char buf[] = "arg1, arg2"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeArguments(line); + auto token1 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token1, "arg1"_st); EXPECT_EQ(line, buf + sizeof("arg1,") - 1); - auto token2 = pfConsoleEngine::TokenizeArguments(line); + auto token2 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token2, "arg2"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token3 = pfConsoleEngine::TokenizeArguments(line); + auto token3 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_FALSE(token3); EXPECT_EQ(line, buf + sizeof(buf) - 1); } -TEST(pfConsoleCore, TokenizeArgumentsPairMixedQuotes) +TEST(pfConsoleTokenizer, TokenizeArgumentsPairMixedQuotes) { const char buf[] = "\"argument '1'\" 'argument \"2\"'"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeArguments(line); + auto token1 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token1, "argument '1'"_st); EXPECT_EQ(line, buf + sizeof("\"argument '1'\"") - 1); - auto token2 = pfConsoleEngine::TokenizeArguments(line); + auto token2 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token2, "argument \"2\""_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token3 = pfConsoleEngine::TokenizeArguments(line); + auto token3 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_FALSE(token3); EXPECT_EQ(line, buf + sizeof(buf) - 1); } // Tokenize arguments, 3 tokens -TEST(pfConsoleCore, TokenizeArgumentsTriple) +TEST(pfConsoleTokenizer, TokenizeArgumentsTriple) { const char buf[] = "1.2 3.4 5.6"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeArguments(line); + auto token1 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token1, "1.2"_st); EXPECT_EQ(line, buf + sizeof("1.2 ") - 1); - auto token2 = pfConsoleEngine::TokenizeArguments(line); + auto token2 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token2, "3.4"_st); EXPECT_EQ(line, buf + sizeof("1.2 3.4 ") - 1); - auto token3 = pfConsoleEngine::TokenizeArguments(line); + auto token3 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token3, "5.6"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token4 = pfConsoleEngine::TokenizeArguments(line); + auto token4 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_FALSE(token4); EXPECT_EQ(line, buf + sizeof(buf) - 1); } -TEST(pfConsoleCore, TokenizeArgumentsTripleCommas) +TEST(pfConsoleTokenizer, TokenizeArgumentsTripleCommas) { const char buf[] = "1.2, 3.4, 5.6"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeArguments(line); + auto token1 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token1, "1.2"_st); EXPECT_EQ(line, buf + sizeof("1.2,") - 1); - auto token2 = pfConsoleEngine::TokenizeArguments(line); + auto token2 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token2, "3.4"_st); EXPECT_EQ(line, buf + sizeof("1.2, 3.4,") - 1); - auto token3 = pfConsoleEngine::TokenizeArguments(line); + auto token3 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token3, "5.6"_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token4 = pfConsoleEngine::TokenizeArguments(line); + auto token4 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_FALSE(token4); EXPECT_EQ(line, buf + sizeof(buf) - 1); } -TEST(pfConsoleCore, TokenizeArgumentsTripleEmptyQuotes) +TEST(pfConsoleTokenizer, TokenizeArgumentsTripleEmptyQuotes) { const char buf[] = "'' \"\" ''"; const char* line = buf; - auto token1 = pfConsoleEngine::TokenizeArguments(line); + auto token1 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token1, ""_st); EXPECT_EQ(line, buf + sizeof("''") - 1); - auto token2 = pfConsoleEngine::TokenizeArguments(line); + auto token2 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token2, ""_st); EXPECT_EQ(line, buf + sizeof("'' \"\"") - 1); - auto token3 = pfConsoleEngine::TokenizeArguments(line); + auto token3 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_EQ(token3, ""_st); EXPECT_EQ(line, buf + sizeof(buf) - 1); - auto token4 = pfConsoleEngine::TokenizeArguments(line); + auto token4 = pfConsoleTokenizer::TokenizeArguments(line); EXPECT_FALSE(token4); EXPECT_EQ(line, buf + sizeof(buf) - 1); } From 7970f453bf827a77198119451e1ae11ff690b9fc Mon Sep 17 00:00:00 2001 From: dgelessus Date: Sun, 30 Jul 2023 03:30:49 +0200 Subject: [PATCH 08/17] Refactor pfConsoleTokenizer to be stateful This avoids having to pass a non-const reference into every call and allows signaling errors in a less dirty way. --- .../pfConsoleCore/pfConsoleEngine.cpp | 40 +-- .../pfConsoleCore/pfConsoleParser.cpp | 44 ++- .../pfConsoleCore/pfConsoleParser.h | 23 +- .../test_pfConsoleTokenizer.cpp | 310 ++++++++++-------- 4 files changed, 218 insertions(+), 199 deletions(-) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp index b33f64327e..df3450e987 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp @@ -77,11 +77,11 @@ bool pfConsoleEngine::PrintCmdHelp(const ST::string& name, void (*PrintFn)(const { pfConsoleCmd *cmd; pfConsoleCmdGroup *group, *subGrp; - const char* namePtr = name.c_str(); /// Scan for subgroups. This can be an empty loop + pfConsoleTokenizer tokenizer(name.c_str()); group = pfConsoleCmdGroup::GetBaseGroup(); - auto token = pfConsoleTokenizer::TokenizeCommandName(namePtr); + auto token = tokenizer.NextNamePart(); while (token) { // Take this token and check to see if it's a group if ((subGrp = group->FindSubGroupNoCase(*token)) != nullptr) @@ -89,7 +89,7 @@ bool pfConsoleEngine::PrintCmdHelp(const ST::string& name, void (*PrintFn)(const else break; - token = pfConsoleTokenizer::TokenizeCommandName(namePtr); + token = tokenizer.NextNamePart(); } if (!token) { @@ -140,11 +140,11 @@ ST::string pfConsoleEngine::GetCmdSignature(const ST::string& name) { pfConsoleCmd *cmd; pfConsoleCmdGroup *group, *subGrp; - const char* namePtr = name.c_str(); /// Scan for subgroups. This can be an empty loop + pfConsoleTokenizer tokenizer(name.c_str()); group = pfConsoleCmdGroup::GetBaseGroup(); - auto token = pfConsoleTokenizer::TokenizeCommandName(namePtr); + auto token = tokenizer.NextNamePart(); while (token) { // Take this token and check to see if it's a group if ((subGrp = group->FindSubGroupNoCase(*token)) != nullptr) @@ -152,7 +152,7 @@ ST::string pfConsoleEngine::GetCmdSignature(const ST::string& name) else break; - token = pfConsoleTokenizer::TokenizeCommandName(namePtr); + token = tokenizer.NextNamePart(); } if (!token) { @@ -226,11 +226,11 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S int32_t numParams, i; pfConsoleCmdParam paramArray[ fMaxNumParams + 1 ]; bool valid = true; - const char* linePtr = line.c_str(); /// Loop #1: Scan for subgroups. This can be an empty loop + pfConsoleTokenizer tokenizer(line.c_str()); group = pfConsoleCmdGroup::GetBaseGroup(); - auto token = pfConsoleTokenizer::TokenizeCommandName(linePtr); + auto token = tokenizer.NextNamePart(); while (token) { // Take this token and check to see if it's a group if ((subGrp = group->FindSubGroupNoCase(*token)) != nullptr) @@ -238,7 +238,7 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S else break; - token = pfConsoleTokenizer::TokenizeCommandName(linePtr); + token = tokenizer.NextNamePart(); } if (!token) { @@ -259,14 +259,9 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S /// params for (numParams = 0; numParams < fMaxNumParams - && (token = pfConsoleTokenizer::TokenizeArguments(linePtr)) + && (token = tokenizer.NextArgument()) && valid; numParams++ ) { - if (*token == pfConsoleTokenizer::kTokenizeError) { - fErrorMsg = ST_LITERAL("Invalid syntax: unterminated quoted parameter"); - return false; - } - // Special case for context variables--if we're specifying one, we want to just grab // the value of it and return that instead valid = false; @@ -289,6 +284,12 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S if( !valid ) valid = IConvertToParam(cmd->GetSigEntry(numParams), *token, ¶mArray[numParams]); } + + if (!tokenizer.fErrorMsg.empty()) { + fErrorMsg = ST::format("Invalid syntax: {}", tokenizer.fErrorMsg); + return false; + } + for( i = numParams; i < fMaxNumParams + 1; i++ ) paramArray[ i ].SetNone(); @@ -378,11 +379,11 @@ ST::string pfConsoleEngine::FindPartialCmd(const ST::string& line, bool findAgai /// New search ST::string_stream newStr; - const char* linePtr = line.c_str(); /// Loop #1: Scan for subgroups. This can be an empty loop + pfConsoleTokenizer tokenizer(line.c_str()); pfConsoleCmdGroup* group = pfConsoleCmdGroup::GetBaseGroup(); - auto token = pfConsoleTokenizer::TokenizeCommandName(linePtr); + auto token = tokenizer.NextNamePart(); while (token) { // Take this token and check to see if it's a group pfConsoleCmdGroup* subGrp = group->FindSubGroupNoCase(*token, 0, nullptr); @@ -392,7 +393,7 @@ ST::string pfConsoleEngine::FindPartialCmd(const ST::string& line, bool findAgai group = subGrp; newStr << group->GetName() << '.'; - token = pfConsoleTokenizer::TokenizeCommandName(linePtr); + token = tokenizer.NextNamePart(); } if (token) { @@ -418,8 +419,7 @@ ST::string pfConsoleEngine::FindPartialCmd(const ST::string& line, bool findAgai if( preserveParams ) { /// Preserve the rest of the string after the matched command - if (linePtr != nullptr) - newStr << linePtr; + newStr << tokenizer.fPos; } return newStr.to_string(); diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp index 98d60b6aec..f3077b520b 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp @@ -42,70 +42,66 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include "pfConsoleParser.h" -#include - -// Yes, this is invalid UTF-8. Yes, this is awful. -const ST::string pfConsoleTokenizer::kTokenizeError = ST_LITERAL("\xff"); - static const char kTokenSeparators[] = " =\r\n\t,"; static const char kTokenGrpSeps[] = " =\r\n._\t,"; -//WARNING: Potentially increments the pointer passed to it. -std::optional pfConsoleTokenizer::TokenizeCommandName(const char*& line) +std::optional pfConsoleTokenizer::NextNamePart() { - const char* begin = line; + const char* begin = fPos; while (*begin && isspace(static_cast(*begin))) ++begin; - for (line = begin; *line; ++line) { + for (fPos = begin; *fPos; ++fPos) { for (const char *sep = kTokenGrpSeps; *sep; ++sep) { - if (*line == *sep) { - const char* end = line; - while (*++line && (*line == *sep)) + if (*fPos == *sep) { + const char* end = fPos; + while (*++fPos && (*fPos == *sep)) /* skip duplicate delimiters */; return ST::string::from_utf8(begin, end - begin); } } } - if (begin == line) { + if (begin == fPos) { + fErrorMsg.clear(); return {}; } return begin; } -//WARNING: Potentially increments the pointer passed to it. -std::optional pfConsoleTokenizer::TokenizeArguments(const char*& line) +std::optional pfConsoleTokenizer::NextArgument() { - const char* begin = line; + const char* begin = fPos; while (*begin && isspace(static_cast(*begin))) ++begin; - for (line = begin; *line; ++line) { + for (fPos = begin; *fPos; ++fPos) { if (*begin == '"' || *begin == '\'') { // Handle strings as a single token ++begin; - const char* end = strchr(begin, *line); + const char* end = strchr(begin, *fPos); if (end == nullptr) { - return kTokenizeError; + fErrorMsg = ST_LITERAL("unterminated quoted parameter"); + return {}; } - line = end + 1; + fPos = end + 1; return ST::string::from_utf8(begin, end - begin); } for (const char *sep = kTokenSeparators; *sep; ++sep) { - if (*line == *sep) { - const char* end = line; - while (*++line && (*line == *sep)) + if (*fPos == *sep) { + const char* end = fPos; + while (*++fPos && (*fPos == *sep)) /* skip duplicate delimiters */; return ST::string::from_utf8(begin, end - begin); } } } - if (begin == line) { + if (begin == fPos) { + fErrorMsg.clear(); return {}; } diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h index 675e325dff..3ccfdd5f13 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h @@ -46,24 +46,25 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include "HeadSpin.h" #include - -namespace ST { class string; } +#include class pfConsoleTokenizer { public: - // Special value returned by the Tokenize methods (currently only TokenizeArguments) to indicate a syntax error. - static const ST::string kTokenizeError; + const char* fPos; + ST::string fErrorMsg; + + pfConsoleTokenizer(const char* line) : fPos(line), fErrorMsg() {} - // Parse the next command name or argument token from the given input line. + // Parse the next command name or argument token from the input line. // On success, returns the parsed token. // (Note that a successfully parsed token may be an empty string, e. g. from an empty pair of quotes!) - // The line pointer is incremented to point after that token - // so that it can be passed into another call to continue tokenizing. - // If the next token couldn't be parsed (e. g. quote not closed), returns kTokenizeError. - // If there are no further tokens in the line, returns an empty std::optional. - static std::optional TokenizeCommandName(const char*& line); - static std::optional TokenizeArguments(const char*& line); + // If the next token couldn't be parsed (e. g. quote not closed), + // returns an empty std::optional and sets fErrorMsg to a non-empty string. + // If there are no further tokens in the line, + // returns an empty std::optional and sets fErrorMsg to an empty string. + std::optional NextNamePart(); + std::optional NextArgument(); }; #endif // _pfConsolePareser_h diff --git a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleTokenizer.cpp b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleTokenizer.cpp index dd86783a61..80e45801bb 100644 --- a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleTokenizer.cpp +++ b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleTokenizer.cpp @@ -52,29 +52,31 @@ using namespace ST::literals; TEST(pfConsoleTokenizer, TokenizeCommandNameSingle) { const char buf[] = "SampleCmd1"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token1 = tokenizer.NextNamePart(); EXPECT_EQ(token1, "SampleCmd1"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token2 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token2 = tokenizer.NextNamePart(); EXPECT_FALSE(token2); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } TEST(pfConsoleTokenizer, TokenizeCommandNameSingleWhitespace) { const char buf[] = " SampleCmd1 "; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token1 = tokenizer.NextNamePart(); EXPECT_EQ(token1, "SampleCmd1"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token2 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token2 = tokenizer.NextNamePart(); EXPECT_FALSE(token2); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } // Tokenize command name, 2 tokens @@ -82,55 +84,58 @@ TEST(pfConsoleTokenizer, TokenizeCommandNameSingleWhitespace) TEST(pfConsoleTokenizer, TokenizeCommandNameDot) { const char buf[] = "App.Quit"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token1 = tokenizer.NextNamePart(); EXPECT_EQ(token1, "App"_st); - EXPECT_EQ(line, buf + sizeof("App.") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("App.") - 1); - auto token2 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token2 = tokenizer.NextNamePart(); EXPECT_EQ(token2, "Quit"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token3 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token3 = tokenizer.NextNamePart(); EXPECT_FALSE(token3); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } TEST(pfConsoleTokenizer, TokenizeCommandNameUnderscore) { const char buf[] = "App_Quit"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token1 = tokenizer.NextNamePart(); EXPECT_EQ(token1, "App"_st); - EXPECT_EQ(line, buf + sizeof("App_") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("App_") - 1); - auto token2 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token2 = tokenizer.NextNamePart(); EXPECT_EQ(token2, "Quit"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token3 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token3 = tokenizer.NextNamePart(); EXPECT_FALSE(token3); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } TEST(pfConsoleTokenizer, TokenizeCommandNameSpace) { const char buf[] = "App Quit"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token1 = tokenizer.NextNamePart(); EXPECT_EQ(token1, "App"_st); - EXPECT_EQ(line, buf + sizeof("App ") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("App ") - 1); - auto token2 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token2 = tokenizer.NextNamePart(); EXPECT_EQ(token2, "Quit"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token3 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token3 = tokenizer.NextNamePart(); EXPECT_FALSE(token3); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } // Tokenize command name, 3 tokens @@ -138,67 +143,70 @@ TEST(pfConsoleTokenizer, TokenizeCommandNameSpace) TEST(pfConsoleTokenizer, TokenizeCommandNameDots) { const char buf[] = "Graphics.Renderer.SetYon"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token1 = tokenizer.NextNamePart(); EXPECT_EQ(token1, "Graphics"_st); - EXPECT_EQ(line, buf + sizeof("Graphics.") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("Graphics.") - 1); - auto token2 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token2 = tokenizer.NextNamePart(); EXPECT_EQ(token2, "Renderer"_st); - EXPECT_EQ(line, buf + sizeof("Graphics.Renderer.") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("Graphics.Renderer.") - 1); - auto token3 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token3 = tokenizer.NextNamePart(); EXPECT_EQ(token3, "SetYon"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token4 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token4 = tokenizer.NextNamePart(); EXPECT_FALSE(token4); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } TEST(pfConsoleTokenizer, TokenizeCommandNameUnderscores) { const char buf[] = "Graphics_Renderer_SetYon"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token1 = tokenizer.NextNamePart(); EXPECT_EQ(token1, "Graphics"_st); - EXPECT_EQ(line, buf + sizeof("Graphics_") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("Graphics_") - 1); - auto token2 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token2 = tokenizer.NextNamePart(); EXPECT_EQ(token2, "Renderer"_st); - EXPECT_EQ(line, buf + sizeof("Graphics_Renderer_") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("Graphics_Renderer_") - 1); - auto token3 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token3 = tokenizer.NextNamePart(); EXPECT_EQ(token3, "SetYon"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token4 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token4 = tokenizer.NextNamePart(); EXPECT_FALSE(token4); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } TEST(pfConsoleTokenizer, TokenizeCommandNameSpaces) { const char buf[] = "Graphics Renderer SetYon"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token1 = tokenizer.NextNamePart(); EXPECT_EQ(token1, "Graphics"_st); - EXPECT_EQ(line, buf + sizeof("Graphics ") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("Graphics ") - 1); - auto token2 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token2 = tokenizer.NextNamePart(); EXPECT_EQ(token2, "Renderer"_st); - EXPECT_EQ(line, buf + sizeof("Graphics Renderer ") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("Graphics Renderer ") - 1); - auto token3 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token3 = tokenizer.NextNamePart(); EXPECT_EQ(token3, "SetYon"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token4 = pfConsoleTokenizer::TokenizeCommandName(line); + auto token4 = tokenizer.NextNamePart(); EXPECT_FALSE(token4); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } // Tokenize arguments, 1 token @@ -206,89 +214,96 @@ TEST(pfConsoleTokenizer, TokenizeCommandNameSpaces) TEST(pfConsoleTokenizer, TokenizeArgumentsSingle) { const char buf[] = "arg"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeArguments(line); + auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "arg"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token2 = pfConsoleTokenizer::TokenizeArguments(line); + auto token2 = tokenizer.NextArgument(); EXPECT_FALSE(token2); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } TEST(pfConsoleTokenizer, TokenizeArgumentsSingleWhitespace) { const char buf[] = " arg "; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeArguments(line); + auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "arg"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token2 = pfConsoleTokenizer::TokenizeArguments(line); + auto token2 = tokenizer.NextArgument(); EXPECT_FALSE(token2); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } TEST(pfConsoleTokenizer, TokenizeArgumentsSingleUnderscore) { const char buf[] = "arg_test"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeArguments(line); + auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "arg_test"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token2 = pfConsoleTokenizer::TokenizeArguments(line); + auto token2 = tokenizer.NextArgument(); EXPECT_FALSE(token2); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } TEST(pfConsoleTokenizer, TokenizeArgumentsSingleDoubleQuote) { const char buf[] = "\"(Default Device)\""; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeArguments(line); + auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "(Default Device)"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token2 = pfConsoleTokenizer::TokenizeArguments(line); + auto token2 = tokenizer.NextArgument(); EXPECT_FALSE(token2); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } TEST(pfConsoleTokenizer, TokenizeArgumentsSingleSingleQuote) { const char buf[] = "'(Default Device)'"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeArguments(line); + auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "(Default Device)"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token2 = pfConsoleTokenizer::TokenizeArguments(line); + auto token2 = tokenizer.NextArgument(); EXPECT_FALSE(token2); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } TEST(pfConsoleTokenizer, TokenizeArgumentsSingleDoubleQuoteUnclosed) { const char buf[] = "\"(Default Device)"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeArguments(line); - EXPECT_EQ(token1, "\xff"_st); + auto token1 = tokenizer.NextArgument(); + EXPECT_FALSE(token1); + EXPECT_FALSE(tokenizer.fErrorMsg.empty()); } TEST(pfConsoleTokenizer, TokenizeArgumentsSingleSingleQuoteUnclosed) { const char buf[] = "'(Default Device)"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeArguments(line); - EXPECT_EQ(token1, "\xff"_st); + auto token1 = tokenizer.NextArgument(); + EXPECT_FALSE(token1); + EXPECT_FALSE(tokenizer.fErrorMsg.empty()); } // Tokenize arguments, 2 tokens @@ -296,73 +311,77 @@ TEST(pfConsoleTokenizer, TokenizeArgumentsSingleSingleQuoteUnclosed) TEST(pfConsoleTokenizer, TokenizeArgumentsPair) { const char buf[] = "arg1 arg2"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeArguments(line); + auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "arg1"_st); - EXPECT_EQ(line, buf + sizeof("arg1 ") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("arg1 ") - 1); - auto token2 = pfConsoleTokenizer::TokenizeArguments(line); + auto token2 = tokenizer.NextArgument(); EXPECT_EQ(token2, "arg2"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token3 = pfConsoleTokenizer::TokenizeArguments(line); + auto token3 = tokenizer.NextArgument(); EXPECT_FALSE(token3); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } TEST(pfConsoleTokenizer, TokenizeArgumentsPairWhitespace) { const char buf[] = " arg1 arg2 "; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeArguments(line); + auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "arg1"_st); - EXPECT_EQ(line, buf + sizeof(" arg1 ") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(" arg1 ") - 1); - auto token2 = pfConsoleTokenizer::TokenizeArguments(line); + auto token2 = tokenizer.NextArgument(); EXPECT_EQ(token2, "arg2"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token3 = pfConsoleTokenizer::TokenizeArguments(line); + auto token3 = tokenizer.NextArgument(); EXPECT_FALSE(token3); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } TEST(pfConsoleTokenizer, TokenizeArgumentsPairComma) { const char buf[] = "arg1, arg2"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeArguments(line); + auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "arg1"_st); - EXPECT_EQ(line, buf + sizeof("arg1,") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("arg1,") - 1); - auto token2 = pfConsoleTokenizer::TokenizeArguments(line); + auto token2 = tokenizer.NextArgument(); EXPECT_EQ(token2, "arg2"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token3 = pfConsoleTokenizer::TokenizeArguments(line); + auto token3 = tokenizer.NextArgument(); EXPECT_FALSE(token3); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } TEST(pfConsoleTokenizer, TokenizeArgumentsPairMixedQuotes) { const char buf[] = "\"argument '1'\" 'argument \"2\"'"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeArguments(line); + auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "argument '1'"_st); - EXPECT_EQ(line, buf + sizeof("\"argument '1'\"") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("\"argument '1'\"") - 1); - auto token2 = pfConsoleTokenizer::TokenizeArguments(line); + auto token2 = tokenizer.NextArgument(); EXPECT_EQ(token2, "argument \"2\""_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token3 = pfConsoleTokenizer::TokenizeArguments(line); + auto token3 = tokenizer.NextArgument(); EXPECT_FALSE(token3); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } // Tokenize arguments, 3 tokens @@ -370,65 +389,68 @@ TEST(pfConsoleTokenizer, TokenizeArgumentsPairMixedQuotes) TEST(pfConsoleTokenizer, TokenizeArgumentsTriple) { const char buf[] = "1.2 3.4 5.6"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeArguments(line); + auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "1.2"_st); - EXPECT_EQ(line, buf + sizeof("1.2 ") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("1.2 ") - 1); - auto token2 = pfConsoleTokenizer::TokenizeArguments(line); + auto token2 = tokenizer.NextArgument(); EXPECT_EQ(token2, "3.4"_st); - EXPECT_EQ(line, buf + sizeof("1.2 3.4 ") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("1.2 3.4 ") - 1); - auto token3 = pfConsoleTokenizer::TokenizeArguments(line); + auto token3 = tokenizer.NextArgument(); EXPECT_EQ(token3, "5.6"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token4 = pfConsoleTokenizer::TokenizeArguments(line); + auto token4 = tokenizer.NextArgument(); EXPECT_FALSE(token4); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } TEST(pfConsoleTokenizer, TokenizeArgumentsTripleCommas) { const char buf[] = "1.2, 3.4, 5.6"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeArguments(line); + auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "1.2"_st); - EXPECT_EQ(line, buf + sizeof("1.2,") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("1.2,") - 1); - auto token2 = pfConsoleTokenizer::TokenizeArguments(line); + auto token2 = tokenizer.NextArgument(); EXPECT_EQ(token2, "3.4"_st); - EXPECT_EQ(line, buf + sizeof("1.2, 3.4,") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("1.2, 3.4,") - 1); - auto token3 = pfConsoleTokenizer::TokenizeArguments(line); + auto token3 = tokenizer.NextArgument(); EXPECT_EQ(token3, "5.6"_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token4 = pfConsoleTokenizer::TokenizeArguments(line); + auto token4 = tokenizer.NextArgument(); EXPECT_FALSE(token4); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } TEST(pfConsoleTokenizer, TokenizeArgumentsTripleEmptyQuotes) { const char buf[] = "'' \"\" ''"; - const char* line = buf; + pfConsoleTokenizer tokenizer(buf); - auto token1 = pfConsoleTokenizer::TokenizeArguments(line); + auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, ""_st); - EXPECT_EQ(line, buf + sizeof("''") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("''") - 1); - auto token2 = pfConsoleTokenizer::TokenizeArguments(line); + auto token2 = tokenizer.NextArgument(); EXPECT_EQ(token2, ""_st); - EXPECT_EQ(line, buf + sizeof("'' \"\"") - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof("'' \"\"") - 1); - auto token3 = pfConsoleTokenizer::TokenizeArguments(line); + auto token3 = tokenizer.NextArgument(); EXPECT_EQ(token3, ""_st); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); - auto token4 = pfConsoleTokenizer::TokenizeArguments(line); + auto token4 = tokenizer.NextArgument(); EXPECT_FALSE(token4); - EXPECT_EQ(line, buf + sizeof(buf) - 1); + EXPECT_TRUE(tokenizer.fErrorMsg.empty()); + EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); } From ff4807f86521cb6d2b91734127aae9decc610f74 Mon Sep 17 00:00:00 2001 From: dgelessus Date: Sun, 30 Jul 2023 15:22:03 +0200 Subject: [PATCH 09/17] Port pfConsoleTokenizer fully to ST::string Should now handle embedded zero bytes correctly (as if anyone cares...). --- .../pfConsoleCore/pfConsoleEngine.cpp | 10 +- .../pfConsoleCore/pfConsoleParser.cpp | 31 ++- .../pfConsoleCore/pfConsoleParser.h | 8 +- .../test_pfConsoleTokenizer.cpp | 206 +++++++++--------- 4 files changed, 133 insertions(+), 122 deletions(-) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp index df3450e987..594baf7a5b 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp @@ -79,7 +79,7 @@ bool pfConsoleEngine::PrintCmdHelp(const ST::string& name, void (*PrintFn)(const pfConsoleCmdGroup *group, *subGrp; /// Scan for subgroups. This can be an empty loop - pfConsoleTokenizer tokenizer(name.c_str()); + pfConsoleTokenizer tokenizer(name); group = pfConsoleCmdGroup::GetBaseGroup(); auto token = tokenizer.NextNamePart(); while (token) { @@ -142,7 +142,7 @@ ST::string pfConsoleEngine::GetCmdSignature(const ST::string& name) pfConsoleCmdGroup *group, *subGrp; /// Scan for subgroups. This can be an empty loop - pfConsoleTokenizer tokenizer(name.c_str()); + pfConsoleTokenizer tokenizer(name); group = pfConsoleCmdGroup::GetBaseGroup(); auto token = tokenizer.NextNamePart(); while (token) { @@ -228,7 +228,7 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S bool valid = true; /// Loop #1: Scan for subgroups. This can be an empty loop - pfConsoleTokenizer tokenizer(line.c_str()); + pfConsoleTokenizer tokenizer(line); group = pfConsoleCmdGroup::GetBaseGroup(); auto token = tokenizer.NextNamePart(); while (token) { @@ -381,7 +381,7 @@ ST::string pfConsoleEngine::FindPartialCmd(const ST::string& line, bool findAgai ST::string_stream newStr; /// Loop #1: Scan for subgroups. This can be an empty loop - pfConsoleTokenizer tokenizer(line.c_str()); + pfConsoleTokenizer tokenizer(line); pfConsoleCmdGroup* group = pfConsoleCmdGroup::GetBaseGroup(); auto token = tokenizer.NextNamePart(); while (token) { @@ -419,7 +419,7 @@ ST::string pfConsoleEngine::FindPartialCmd(const ST::string& line, bool findAgai if( preserveParams ) { /// Preserve the rest of the string after the matched command - newStr << tokenizer.fPos; + newStr.append(tokenizer.fPos, tokenizer.fEnd - tokenizer.fPos); } return newStr.to_string(); diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp index f3077b520b..a5324584d9 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp @@ -49,15 +49,17 @@ std::optional pfConsoleTokenizer::NextNamePart() { const char* begin = fPos; - while (*begin && isspace(static_cast(*begin))) + while (begin != fEnd && isspace(static_cast(*begin))) ++begin; - for (fPos = begin; *fPos; ++fPos) { + for (fPos = begin; fPos != fEnd; ++fPos) { for (const char *sep = kTokenGrpSeps; *sep; ++sep) { if (*fPos == *sep) { const char* end = fPos; - while (*++fPos && (*fPos == *sep)) - /* skip duplicate delimiters */; + while (fPos != fEnd && (*fPos == *sep)) { + // skip duplicate delimiters + ++fPos; + } return ST::string::from_utf8(begin, end - begin); } } @@ -75,17 +77,20 @@ std::optional pfConsoleTokenizer::NextArgument() { const char* begin = fPos; - while (*begin && isspace(static_cast(*begin))) + while (begin != fEnd && isspace(static_cast(*begin))) ++begin; - for (fPos = begin; *fPos; ++fPos) { + for (fPos = begin; fPos != fEnd; ++fPos) { if (*begin == '"' || *begin == '\'') { // Handle strings as a single token ++begin; - const char* end = strchr(begin, *fPos); - if (end == nullptr) { - fErrorMsg = ST_LITERAL("unterminated quoted parameter"); - return {}; + const char* end = begin; + while (*end != *fPos) { + ++end; + if (end == fEnd) { + fErrorMsg = ST_LITERAL("unterminated quoted parameter"); + return {}; + } } fPos = end + 1; return ST::string::from_utf8(begin, end - begin); @@ -93,8 +98,10 @@ std::optional pfConsoleTokenizer::NextArgument() for (const char *sep = kTokenSeparators; *sep; ++sep) { if (*fPos == *sep) { const char* end = fPos; - while (*++fPos && (*fPos == *sep)) - /* skip duplicate delimiters */; + while (fPos != fEnd && (*fPos == *sep)) { + // skip duplicate delimiters + ++fPos; + } return ST::string::from_utf8(begin, end - begin); } } diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h index 3ccfdd5f13..6ea9b5ce05 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h @@ -51,10 +51,14 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com class pfConsoleTokenizer { public: - const char* fPos; + ST::string::const_iterator fPos; + ST::string::const_iterator fEnd; ST::string fErrorMsg; - pfConsoleTokenizer(const char* line) : fPos(line), fErrorMsg() {} + pfConsoleTokenizer(ST::string::const_iterator begin, ST::string::const_iterator end) : + fPos(begin), fEnd(end), fErrorMsg() + {} + pfConsoleTokenizer(const ST::string& line) : pfConsoleTokenizer(line.begin(), line.end()) {} // Parse the next command name or argument token from the input line. // On success, returns the parsed token. diff --git a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleTokenizer.cpp b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleTokenizer.cpp index 80e45801bb..3752517370 100644 --- a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleTokenizer.cpp +++ b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleTokenizer.cpp @@ -51,245 +51,245 @@ using namespace ST::literals; TEST(pfConsoleTokenizer, TokenizeCommandNameSingle) { - const char buf[] = "SampleCmd1"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "SampleCmd1"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextNamePart(); EXPECT_EQ(token1, "SampleCmd1"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token2 = tokenizer.NextNamePart(); EXPECT_FALSE(token2); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } TEST(pfConsoleTokenizer, TokenizeCommandNameSingleWhitespace) { - const char buf[] = " SampleCmd1 "; - pfConsoleTokenizer tokenizer(buf); + ST::string string = " SampleCmd1 "_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextNamePart(); EXPECT_EQ(token1, "SampleCmd1"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token2 = tokenizer.NextNamePart(); EXPECT_FALSE(token2); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } // Tokenize command name, 2 tokens TEST(pfConsoleTokenizer, TokenizeCommandNameDot) { - const char buf[] = "App.Quit"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "App.Quit"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextNamePart(); EXPECT_EQ(token1, "App"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("App.") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("App.") - 1); auto token2 = tokenizer.NextNamePart(); EXPECT_EQ(token2, "Quit"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token3 = tokenizer.NextNamePart(); EXPECT_FALSE(token3); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } TEST(pfConsoleTokenizer, TokenizeCommandNameUnderscore) { - const char buf[] = "App_Quit"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "App_Quit"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextNamePart(); EXPECT_EQ(token1, "App"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("App_") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("App_") - 1); auto token2 = tokenizer.NextNamePart(); EXPECT_EQ(token2, "Quit"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token3 = tokenizer.NextNamePart(); EXPECT_FALSE(token3); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } TEST(pfConsoleTokenizer, TokenizeCommandNameSpace) { - const char buf[] = "App Quit"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "App Quit"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextNamePart(); EXPECT_EQ(token1, "App"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("App ") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("App ") - 1); auto token2 = tokenizer.NextNamePart(); EXPECT_EQ(token2, "Quit"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token3 = tokenizer.NextNamePart(); EXPECT_FALSE(token3); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } // Tokenize command name, 3 tokens TEST(pfConsoleTokenizer, TokenizeCommandNameDots) { - const char buf[] = "Graphics.Renderer.SetYon"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "Graphics.Renderer.SetYon"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextNamePart(); EXPECT_EQ(token1, "Graphics"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("Graphics.") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("Graphics.") - 1); auto token2 = tokenizer.NextNamePart(); EXPECT_EQ(token2, "Renderer"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("Graphics.Renderer.") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("Graphics.Renderer.") - 1); auto token3 = tokenizer.NextNamePart(); EXPECT_EQ(token3, "SetYon"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token4 = tokenizer.NextNamePart(); EXPECT_FALSE(token4); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } TEST(pfConsoleTokenizer, TokenizeCommandNameUnderscores) { - const char buf[] = "Graphics_Renderer_SetYon"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "Graphics_Renderer_SetYon"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextNamePart(); EXPECT_EQ(token1, "Graphics"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("Graphics_") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("Graphics_") - 1); auto token2 = tokenizer.NextNamePart(); EXPECT_EQ(token2, "Renderer"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("Graphics_Renderer_") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("Graphics_Renderer_") - 1); auto token3 = tokenizer.NextNamePart(); EXPECT_EQ(token3, "SetYon"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token4 = tokenizer.NextNamePart(); EXPECT_FALSE(token4); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } TEST(pfConsoleTokenizer, TokenizeCommandNameSpaces) { - const char buf[] = "Graphics Renderer SetYon"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "Graphics Renderer SetYon"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextNamePart(); EXPECT_EQ(token1, "Graphics"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("Graphics ") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("Graphics ") - 1); auto token2 = tokenizer.NextNamePart(); EXPECT_EQ(token2, "Renderer"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("Graphics Renderer ") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("Graphics Renderer ") - 1); auto token3 = tokenizer.NextNamePart(); EXPECT_EQ(token3, "SetYon"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token4 = tokenizer.NextNamePart(); EXPECT_FALSE(token4); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } // Tokenize arguments, 1 token TEST(pfConsoleTokenizer, TokenizeArgumentsSingle) { - const char buf[] = "arg"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "arg"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "arg"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token2 = tokenizer.NextArgument(); EXPECT_FALSE(token2); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } TEST(pfConsoleTokenizer, TokenizeArgumentsSingleWhitespace) { - const char buf[] = " arg "; - pfConsoleTokenizer tokenizer(buf); + ST::string string = " arg "_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "arg"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token2 = tokenizer.NextArgument(); EXPECT_FALSE(token2); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } TEST(pfConsoleTokenizer, TokenizeArgumentsSingleUnderscore) { - const char buf[] = "arg_test"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "arg_test"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "arg_test"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token2 = tokenizer.NextArgument(); EXPECT_FALSE(token2); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } TEST(pfConsoleTokenizer, TokenizeArgumentsSingleDoubleQuote) { - const char buf[] = "\"(Default Device)\""; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "\"(Default Device)\""_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "(Default Device)"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token2 = tokenizer.NextArgument(); EXPECT_FALSE(token2); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } TEST(pfConsoleTokenizer, TokenizeArgumentsSingleSingleQuote) { - const char buf[] = "'(Default Device)'"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "'(Default Device)'"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "(Default Device)"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token2 = tokenizer.NextArgument(); EXPECT_FALSE(token2); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } TEST(pfConsoleTokenizer, TokenizeArgumentsSingleDoubleQuoteUnclosed) { - const char buf[] = "\"(Default Device)"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "\"(Default Device)"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextArgument(); EXPECT_FALSE(token1); @@ -298,8 +298,8 @@ TEST(pfConsoleTokenizer, TokenizeArgumentsSingleDoubleQuoteUnclosed) TEST(pfConsoleTokenizer, TokenizeArgumentsSingleSingleQuoteUnclosed) { - const char buf[] = "'(Default Device)"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "'(Default Device)"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextArgument(); EXPECT_FALSE(token1); @@ -310,147 +310,147 @@ TEST(pfConsoleTokenizer, TokenizeArgumentsSingleSingleQuoteUnclosed) TEST(pfConsoleTokenizer, TokenizeArgumentsPair) { - const char buf[] = "arg1 arg2"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "arg1 arg2"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "arg1"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("arg1 ") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("arg1 ") - 1); auto token2 = tokenizer.NextArgument(); EXPECT_EQ(token2, "arg2"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token3 = tokenizer.NextArgument(); EXPECT_FALSE(token3); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } TEST(pfConsoleTokenizer, TokenizeArgumentsPairWhitespace) { - const char buf[] = " arg1 arg2 "; - pfConsoleTokenizer tokenizer(buf); + ST::string string = " arg1 arg2 "_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "arg1"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(" arg1 ") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof(" arg1 ") - 1); auto token2 = tokenizer.NextArgument(); EXPECT_EQ(token2, "arg2"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token3 = tokenizer.NextArgument(); EXPECT_FALSE(token3); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } TEST(pfConsoleTokenizer, TokenizeArgumentsPairComma) { - const char buf[] = "arg1, arg2"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "arg1, arg2"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "arg1"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("arg1,") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("arg1,") - 1); auto token2 = tokenizer.NextArgument(); EXPECT_EQ(token2, "arg2"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token3 = tokenizer.NextArgument(); EXPECT_FALSE(token3); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } TEST(pfConsoleTokenizer, TokenizeArgumentsPairMixedQuotes) { - const char buf[] = "\"argument '1'\" 'argument \"2\"'"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "\"argument '1'\" 'argument \"2\"'"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "argument '1'"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("\"argument '1'\"") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("\"argument '1'\"") - 1); auto token2 = tokenizer.NextArgument(); EXPECT_EQ(token2, "argument \"2\""_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token3 = tokenizer.NextArgument(); EXPECT_FALSE(token3); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } // Tokenize arguments, 3 tokens TEST(pfConsoleTokenizer, TokenizeArgumentsTriple) { - const char buf[] = "1.2 3.4 5.6"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "1.2 3.4 5.6"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "1.2"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("1.2 ") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("1.2 ") - 1); auto token2 = tokenizer.NextArgument(); EXPECT_EQ(token2, "3.4"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("1.2 3.4 ") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("1.2 3.4 ") - 1); auto token3 = tokenizer.NextArgument(); EXPECT_EQ(token3, "5.6"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token4 = tokenizer.NextArgument(); EXPECT_FALSE(token4); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } TEST(pfConsoleTokenizer, TokenizeArgumentsTripleCommas) { - const char buf[] = "1.2, 3.4, 5.6"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "1.2, 3.4, 5.6"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, "1.2"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("1.2,") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("1.2,") - 1); auto token2 = tokenizer.NextArgument(); EXPECT_EQ(token2, "3.4"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("1.2, 3.4,") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("1.2, 3.4,") - 1); auto token3 = tokenizer.NextArgument(); EXPECT_EQ(token3, "5.6"_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token4 = tokenizer.NextArgument(); EXPECT_FALSE(token4); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } TEST(pfConsoleTokenizer, TokenizeArgumentsTripleEmptyQuotes) { - const char buf[] = "'' \"\" ''"; - pfConsoleTokenizer tokenizer(buf); + ST::string string = "'' \"\" ''"_st; + pfConsoleTokenizer tokenizer(string); auto token1 = tokenizer.NextArgument(); EXPECT_EQ(token1, ""_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("''") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("''") - 1); auto token2 = tokenizer.NextArgument(); EXPECT_EQ(token2, ""_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof("'' \"\"") - 1); + EXPECT_EQ(tokenizer.fPos, string.begin() + sizeof("'' \"\"") - 1); auto token3 = tokenizer.NextArgument(); EXPECT_EQ(token3, ""_st); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); auto token4 = tokenizer.NextArgument(); EXPECT_FALSE(token4); EXPECT_TRUE(tokenizer.fErrorMsg.empty()); - EXPECT_EQ(tokenizer.fPos, buf + sizeof(buf) - 1); + EXPECT_EQ(tokenizer.fPos, string.end()); } From 5f2b03483be360e05c9fbb28599a925b7c120926 Mon Sep 17 00:00:00 2001 From: dgelessus Date: Sun, 30 Jul 2023 19:16:12 +0200 Subject: [PATCH 10/17] Refactor console command name parsing and group lookup These have to be coupled, because command name parts can also be separated using spaces instead of dots/underscores. --- .../pfConsoleCore/pfConsoleEngine.cpp | 89 ++----- .../pfConsoleCore/pfConsoleParser.cpp | 19 ++ .../pfConsoleCore/pfConsoleParser.h | 20 ++ .../pfConsoleCoreTest/CMakeLists.txt | 1 + .../test_pfConsoleParser.cpp | 218 ++++++++++++++++++ 5 files changed, 281 insertions(+), 66 deletions(-) create mode 100644 Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleParser.cpp diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp index 594baf7a5b..fbd8580273 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp @@ -76,36 +76,18 @@ pfConsoleEngine::~pfConsoleEngine() bool pfConsoleEngine::PrintCmdHelp(const ST::string& name, void (*PrintFn)(const ST::string&)) { pfConsoleCmd *cmd; - pfConsoleCmdGroup *group, *subGrp; - - /// Scan for subgroups. This can be an empty loop - pfConsoleTokenizer tokenizer(name); - group = pfConsoleCmdGroup::GetBaseGroup(); - auto token = tokenizer.NextNamePart(); - while (token) { - // Take this token and check to see if it's a group - if ((subGrp = group->FindSubGroupNoCase(*token)) != nullptr) - group = subGrp; - else - break; - token = tokenizer.NextNamePart(); - } + pfConsoleParser parser(name); + auto [group, token] = parser.ParseGroupAndName(); if (!token) { - if (group == nullptr) - { - fErrorMsg = ST_LITERAL("Invalid command syntax"); - return false; - } - // Print help for this group if( group == pfConsoleCmdGroup::GetBaseGroup() ) PrintFn(ST_LITERAL("Base commands and groups:")); else PrintFn(ST::format("Group {}:", group->GetName())); PrintFn(ST_LITERAL(" Subgroups:")); - for (subGrp = group->GetFirstSubGroup(); subGrp != nullptr; subGrp = subGrp->GetNext()) + for (pfConsoleCmdGroup* subGrp = group->GetFirstSubGroup(); subGrp != nullptr; subGrp = subGrp->GetNext()) { PrintFn(ST::format(" {}", subGrp->GetName())); } @@ -139,21 +121,9 @@ bool pfConsoleEngine::PrintCmdHelp(const ST::string& name, void (*PrintFn)(const ST::string pfConsoleEngine::GetCmdSignature(const ST::string& name) { pfConsoleCmd *cmd; - pfConsoleCmdGroup *group, *subGrp; - /// Scan for subgroups. This can be an empty loop - pfConsoleTokenizer tokenizer(name); - group = pfConsoleCmdGroup::GetBaseGroup(); - auto token = tokenizer.NextNamePart(); - while (token) { - // Take this token and check to see if it's a group - if ((subGrp = group->FindSubGroupNoCase(*token)) != nullptr) - group = subGrp; - else - break; - - token = tokenizer.NextNamePart(); - } + pfConsoleParser parser(name); + auto [group, token] = parser.ParseGroupAndName(); if (!token) { fErrorMsg = ST_LITERAL("Invalid command syntax"); @@ -222,24 +192,12 @@ bool pfConsoleEngine::ExecuteFile(const plFileName &fileName) bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const ST::string&)) { pfConsoleCmd *cmd; - pfConsoleCmdGroup *group, *subGrp; int32_t numParams, i; pfConsoleCmdParam paramArray[ fMaxNumParams + 1 ]; bool valid = true; - /// Loop #1: Scan for subgroups. This can be an empty loop - pfConsoleTokenizer tokenizer(line); - group = pfConsoleCmdGroup::GetBaseGroup(); - auto token = tokenizer.NextNamePart(); - while (token) { - // Take this token and check to see if it's a group - if ((subGrp = group->FindSubGroupNoCase(*token)) != nullptr) - group = subGrp; - else - break; - - token = tokenizer.NextNamePart(); - } + pfConsoleParser parser(line); + auto [group, token] = parser.ParseGroupAndName(); if (!token) { fErrorMsg = ST_LITERAL("Invalid command syntax"); @@ -259,7 +217,7 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S /// params for (numParams = 0; numParams < fMaxNumParams - && (token = tokenizer.NextArgument()) + && (token = parser.fTokenizer.NextArgument()) && valid; numParams++ ) { // Special case for context variables--if we're specifying one, we want to just grab @@ -285,8 +243,9 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S valid = IConvertToParam(cmd->GetSigEntry(numParams), *token, ¶mArray[numParams]); } - if (!tokenizer.fErrorMsg.empty()) { - fErrorMsg = ST::format("Invalid syntax: {}", tokenizer.fErrorMsg); + ST::string errorMsg = parser.fTokenizer.fErrorMsg; + if (!errorMsg.empty()) { + fErrorMsg = ST::format("Invalid syntax: {}", errorMsg); return false; } @@ -380,20 +339,18 @@ ST::string pfConsoleEngine::FindPartialCmd(const ST::string& line, bool findAgai /// New search ST::string_stream newStr; - /// Loop #1: Scan for subgroups. This can be an empty loop - pfConsoleTokenizer tokenizer(line); - pfConsoleCmdGroup* group = pfConsoleCmdGroup::GetBaseGroup(); - auto token = tokenizer.NextNamePart(); - while (token) { - // Take this token and check to see if it's a group - pfConsoleCmdGroup* subGrp = group->FindSubGroupNoCase(*token, 0, nullptr); - if (subGrp == nullptr) { - break; - } + pfConsoleParser parser(line); + auto [group, token] = parser.ParseGroupAndName(); - group = subGrp; - newStr << group->GetName() << '.'; - token = tokenizer.NextNamePart(); + // Add group name to replacement line. + std::vector reverseParts; + pfConsoleCmdGroup* parent = group; + while (parent != nullptr && parent != pfConsoleCmdGroup::GetBaseGroup()) { + reverseParts.emplace_back(parent->GetName()); + parent = parent->GetParent(); + } + for (auto it = reverseParts.crbegin(); it != reverseParts.crend(); ++it) { + newStr << *it << '.'; } if (token) { @@ -419,7 +376,7 @@ ST::string pfConsoleEngine::FindPartialCmd(const ST::string& line, bool findAgai if( preserveParams ) { /// Preserve the rest of the string after the matched command - newStr.append(tokenizer.fPos, tokenizer.fEnd - tokenizer.fPos); + newStr.append(parser.fTokenizer.fPos, parser.fTokenizer.fEnd - parser.fTokenizer.fPos); } return newStr.to_string(); diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp index a5324584d9..05b9715712 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp @@ -42,6 +42,8 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include "pfConsoleParser.h" +#include "pfConsoleCmd.h" + static const char kTokenSeparators[] = " =\r\n\t,"; static const char kTokenGrpSeps[] = " =\r\n._\t,"; @@ -114,3 +116,20 @@ std::optional pfConsoleTokenizer::NextArgument() return begin; } + +std::pair> pfConsoleParser::ParseGroupAndName() +{ + pfConsoleCmdGroup* group = pfConsoleCmdGroup::GetBaseGroup(); + auto token = fTokenizer.NextNamePart(); + while (token) { + // Take this token and check to see if it's a group + pfConsoleCmdGroup* subGrp = group->FindSubGroupNoCase(*token); + if (subGrp == nullptr) { + return {group, token}; + } + group = subGrp; + token = fTokenizer.NextNamePart(); + } + + return {group, token}; +} diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h index 6ea9b5ce05..c15d66a956 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h @@ -47,6 +47,10 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include #include +#include +#include + +class pfConsoleCmdGroup; class pfConsoleTokenizer { @@ -71,4 +75,20 @@ class pfConsoleTokenizer std::optional NextArgument(); }; +class pfConsoleParser +{ +public: + pfConsoleTokenizer fTokenizer; + + pfConsoleParser(ST::string::const_iterator begin, ST::string::const_iterator end) : fTokenizer(begin, end) {} + pfConsoleParser(const ST::string& line) : pfConsoleParser(line.begin(), line.end()) {} + + // Parse the command name part of the line as far as possible. + // This consumes name part tokens and uses them to look up a command group + // until a token is encountered that isn't a known group name. + // Returns the found group and the first non-group token + // (which may be an empty std::optional if the end of the line was reached). + std::pair> ParseGroupAndName(); +}; + #endif // _pfConsolePareser_h diff --git a/Sources/Tests/FeatureTests/pfConsoleCoreTest/CMakeLists.txt b/Sources/Tests/FeatureTests/pfConsoleCoreTest/CMakeLists.txt index 4676bb3871..c71cbeb391 100644 --- a/Sources/Tests/FeatureTests/pfConsoleCoreTest/CMakeLists.txt +++ b/Sources/Tests/FeatureTests/pfConsoleCoreTest/CMakeLists.txt @@ -1,4 +1,5 @@ set(pfConsoleCore_SOURCES + test_pfConsoleParser.cpp test_pfConsoleTokenizer.cpp ) diff --git a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleParser.cpp b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleParser.cpp new file mode 100644 index 0000000000..2bafebbf3c --- /dev/null +++ b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleParser.cpp @@ -0,0 +1,218 @@ +/*==LICENSE==* + +CyanWorlds.com Engine - MMOG client, server and tools +Copyright (C) 2011 Cyan Worlds, Inc. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +Additional permissions under GNU GPL version 3 section 7 + +If you modify this Program, or any covered work, by linking or +combining it with any of RAD Game Tools Bink SDK, Autodesk 3ds Max SDK, +NVIDIA PhysX SDK, Microsoft DirectX SDK, OpenSSL library, Independent +JPEG Group JPEG library, Microsoft Windows Media SDK, or Apple QuickTime SDK +(or a modified version of those libraries), +containing parts covered by the terms of the Bink SDK EULA, 3ds Max EULA, +PhysX SDK EULA, DirectX SDK EULA, OpenSSL and SSLeay licenses, IJG +JPEG Library README, Windows Media SDK EULA, or QuickTime SDK EULA, the +licensors of this Program grant you additional +permission to convey the resulting work. Corresponding Source for a +non-source form of such a combination shall include the source code for +the parts of OpenSSL and IJG JPEG Library used as well as that of the covered +work. + +You can contact Cyan Worlds, Inc. by email legal@cyan.com + or by snail mail at: + Cyan Worlds, Inc. + 14617 N Newport Hwy + Mead, WA 99021 + +*==LICENSE==*/ + +#include +#include + +#include "pfConsoleCore/pfConsoleCmd.h" +#include "pfConsoleCore/pfConsoleParser.h" + +using namespace ST::literals; + +// Define a few test groups and commands so that we don't have to pull in +// all the real console commands just to test the parsing. + +PF_CONSOLE_BASE_CMD(TestBaseCmd, "", "") +{} + +PF_CONSOLE_GROUP(TestGroup) + +PF_CONSOLE_CMD(TestGroup, SubCmd, "", "") +{} + +PF_CONSOLE_SUBGROUP(TestGroup, SubGroup) + +PF_CONSOLE_CMD(TestGroup_SubGroup, SubSubCmd, "", "") +{} + +// Top-level command outside of any group + +TEST(pfConsoleParser, ParseBaseCommand) +{ + ST::string string = "TestBaseCmd"_st; + pfConsoleParser parser(string); + + auto [group, token] = parser.ParseGroupAndName(); + EXPECT_EQ(group, pfConsoleCmdGroup::GetBaseGroup()); + EXPECT_EQ(token, "TestBaseCmd"_st); + EXPECT_EQ(parser.fTokenizer.fPos, string.end()); +} + +TEST(pfConsoleParser, ParseBaseCommandArgs) +{ + ST::string string = "TestBaseCmd arg1 arg2"_st; + pfConsoleParser parser(string); + + auto [group, token] = parser.ParseGroupAndName(); + EXPECT_EQ(group, pfConsoleCmdGroup::GetBaseGroup()); + EXPECT_EQ(token, "TestBaseCmd"_st); + EXPECT_EQ(parser.fTokenizer.fPos, string.begin() + sizeof("TestBaseCmd ") - 1); +} + +// Top-level group + +TEST(pfConsoleParser, ParseBaseGroup) +{ + ST::string string = "TestGroup"_st; + pfConsoleParser parser(string); + + auto [group, token] = parser.ParseGroupAndName(); + EXPECT_EQ(group, &conGroup_TestGroup); + EXPECT_FALSE(token); + EXPECT_EQ(parser.fTokenizer.fPos, string.end()); +} + +// Command inside top-level group + +TEST(pfConsoleParser, ParseSubCommand) +{ + ST::string string = "TestGroup.SubCmd"_st; + pfConsoleParser parser(string); + + auto [group, token] = parser.ParseGroupAndName(); + EXPECT_EQ(group, &conGroup_TestGroup); + EXPECT_EQ(token, "SubCmd"_st); + EXPECT_EQ(parser.fTokenizer.fPos, string.end()); +} + +TEST(pfConsoleParser, ParseSubCommandArgs) +{ + ST::string string = "TestGroup.SubCmd arg1 arg2"_st; + pfConsoleParser parser(string); + + auto [group, token] = parser.ParseGroupAndName(); + EXPECT_EQ(group, &conGroup_TestGroup); + EXPECT_EQ(token, "SubCmd"_st); + EXPECT_EQ(parser.fTokenizer.fPos, string.begin() + sizeof("TestGroup.SubCmd ") - 1); +} + +TEST(pfConsoleParser, ParseSubCommandSpace) +{ + ST::string string = "TestGroup SubCmd"_st; + pfConsoleParser parser(string); + + auto [group, token] = parser.ParseGroupAndName(); + EXPECT_EQ(group, &conGroup_TestGroup); + EXPECT_EQ(token, "SubCmd"_st); + EXPECT_EQ(parser.fTokenizer.fPos, string.end()); +} + +TEST(pfConsoleParser, ParseSubCommandSpaceArgs) +{ + ST::string string = "TestGroup SubCmd arg1 arg2"_st; + pfConsoleParser parser(string); + + auto [group, token] = parser.ParseGroupAndName(); + EXPECT_EQ(group, &conGroup_TestGroup); + EXPECT_EQ(token, "SubCmd"_st); + EXPECT_EQ(parser.fTokenizer.fPos, string.begin() + sizeof("TestGroup SubCmd ") - 1); +} + +// Subgroup inside other group + +TEST(pfConsoleParser, ParseSubGroup) +{ + ST::string string = "TestGroup.SubGroup"_st; + pfConsoleParser parser(string); + + auto [group, token] = parser.ParseGroupAndName(); + EXPECT_EQ(group, &conGroup_TestGroup_SubGroup); + EXPECT_FALSE(token); + EXPECT_EQ(parser.fTokenizer.fPos, string.end()); +} + +TEST(pfConsoleParser, ParseSubGroupSpace) +{ + ST::string string = "TestGroup SubGroup"_st; + pfConsoleParser parser(string); + + auto [group, token] = parser.ParseGroupAndName(); + EXPECT_EQ(group, &conGroup_TestGroup_SubGroup); + EXPECT_FALSE(token); + EXPECT_EQ(parser.fTokenizer.fPos, string.end()); +} + +// Command inside subgroup + +TEST(pfConsoleParser, ParseSubSubCommand) +{ + ST::string string = "TestGroup.SubGroup.SubSubCmd"_st; + pfConsoleParser parser(string); + + auto [group, token] = parser.ParseGroupAndName(); + EXPECT_EQ(group, &conGroup_TestGroup_SubGroup); + EXPECT_EQ(token, "SubSubCmd"_st); + EXPECT_EQ(parser.fTokenizer.fPos, string.end()); +} + +TEST(pfConsoleParser, ParseSubSubCommandArgs) +{ + ST::string string = "TestGroup.SubGroup.SubSubCmd arg1 arg2"_st; + pfConsoleParser parser(string); + + auto [group, token] = parser.ParseGroupAndName(); + EXPECT_EQ(group, &conGroup_TestGroup_SubGroup); + EXPECT_EQ(token, "SubSubCmd"_st); + EXPECT_EQ(parser.fTokenizer.fPos, string.begin() + sizeof("TestGroup.SubGroup.SubSubCmd ") - 1); +} + +TEST(pfConsoleParser, ParseSubSubCommandSpaces) +{ + ST::string string = "TestGroup SubGroup SubSubCmd"_st; + pfConsoleParser parser(string); + + auto [group, token] = parser.ParseGroupAndName(); + EXPECT_EQ(group, &conGroup_TestGroup_SubGroup); + EXPECT_EQ(token, "SubSubCmd"_st); + EXPECT_EQ(parser.fTokenizer.fPos, string.end()); +} + +TEST(pfConsoleParser, ParseSubSubCommandSpacesArgs) +{ + ST::string string = "TestGroup SubGroup SubSubCmd arg1 arg2"_st; + pfConsoleParser parser(string); + + auto [group, token] = parser.ParseGroupAndName(); + EXPECT_EQ(group, &conGroup_TestGroup_SubGroup); + EXPECT_EQ(token, "SubSubCmd"_st); + EXPECT_EQ(parser.fTokenizer.fPos, string.begin() + sizeof("TestGroup SubGroup SubSubCmd ") - 1); +} From 018dfa96746c6ef37c9e6eaeb9066549ac7a65db Mon Sep 17 00:00:00 2001 From: dgelessus Date: Mon, 31 Jul 2023 22:02:51 +0200 Subject: [PATCH 11/17] Refactor getting full names of console commands/groups --- .../FeatureLib/pfConsoleCore/pfConsoleCmd.cpp | 19 +++++++++++++ .../FeatureLib/pfConsoleCore/pfConsoleCmd.h | 2 ++ .../pfConsoleCore/pfConsoleEngine.cpp | 27 +++---------------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleCmd.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleCmd.cpp index 9b406fac79..1f32908bf4 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleCmd.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleCmd.cpp @@ -47,6 +47,7 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include "pfConsoleCmd.h" +#include #include ////////////////////////////////////////////////////////////////////////////// @@ -134,6 +135,15 @@ void pfConsoleCmdGroup::AddSubGroup( pfConsoleCmdGroup *group ) fBaseCmdGroupRef++; } +ST::string pfConsoleCmdGroup::GetFullName() +{ + if (fParentGroup == nullptr || fParentGroup == GetBaseGroup()) { + return fName; + } else { + return ST::format("{}.{}", fParentGroup->GetFullName(), fName); + } +} + //// FindCommand ///////////////////////////////////////////////////////////// // No longer recursive. @@ -446,6 +456,15 @@ void pfConsoleCmd::Unlink() *fPrevPtr = fNext; } +ST::string pfConsoleCmd::GetFullName() +{ + if (fParentGroup == pfConsoleCmdGroup::GetBaseGroup()) { + return fName; + } else { + return ST::format("{}.{}", fParentGroup->GetFullName(), fName); + } +} + //// GetSigEntry ///////////////////////////////////////////////////////////// uint8_t pfConsoleCmd::GetSigEntry(size_t i) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleCmd.h b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleCmd.h index ad660321d6..70d6ebd6f7 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleCmd.h +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleCmd.h @@ -95,6 +95,7 @@ class pfConsoleCmdGroup pfConsoleCmdGroup *GetNext() { return fNext; } ST::string GetName() { return fName; } + ST::string GetFullName(); pfConsoleCmdGroup *GetParent() { return fParentGroup; } static pfConsoleCmdGroup *GetBaseGroup(); @@ -262,6 +263,7 @@ class pfConsoleCmd pfConsoleCmd *GetNext() { return fNext; } ST::string GetName() { return fName; } + ST::string GetFullName(); ST::string GetHelp() { return fHelpString; } ST::string GetSignature(); diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp index fbd8580273..1d5e8a07a5 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp @@ -343,14 +343,8 @@ ST::string pfConsoleEngine::FindPartialCmd(const ST::string& line, bool findAgai auto [group, token] = parser.ParseGroupAndName(); // Add group name to replacement line. - std::vector reverseParts; - pfConsoleCmdGroup* parent = group; - while (parent != nullptr && parent != pfConsoleCmdGroup::GetBaseGroup()) { - reverseParts.emplace_back(parent->GetName()); - parent = parent->GetParent(); - } - for (auto it = reverseParts.crbegin(); it != reverseParts.crend(); ++it) { - newStr << *it << '.'; + if (group != pfConsoleCmdGroup::GetBaseGroup()) { + newStr << group->GetFullName() << '.'; } if (token) { @@ -395,23 +389,8 @@ ST::string pfConsoleEngine::FindNestedPartialCmd(const ST::string& line, uint32_ if (cmd == nullptr) return {}; - /// Recurse back up and get the group hierarchy - std::vector reverseParts {cmd->GetName()}; - pfConsoleCmdGroup* group = cmd->GetParent(); - while (group != nullptr && group != pfConsoleCmdGroup::GetBaseGroup()) { - reverseParts.emplace_back(group->GetName()); - group = group->GetParent(); - } - ST::string_stream name; - for (auto it = reverseParts.crbegin(); it != reverseParts.crend(); ++it) { - name << *it; - if (it + 1 == reverseParts.crend()) { - name << ' '; - } else { - name << '.'; - } - } + name << cmd->GetFullName() << ' '; if( preserveParams ) { From 28b65f41c116a6758e23d7b61a651fd075e58126 Mon Sep 17 00:00:00 2001 From: dgelessus Date: Mon, 31 Jul 2023 23:03:14 +0200 Subject: [PATCH 12/17] Refactor console command parsing and lookup further where possible Anything that needs possibly partial or ambiguous lookups (help, tab completion) still needs the complex API that parses "as far as possible". --- .../pfConsoleCore/pfConsoleEngine.cpp | 21 ++------ .../pfConsoleCore/pfConsoleParser.cpp | 9 ++++ .../pfConsoleCore/pfConsoleParser.h | 6 +++ .../test_pfConsoleParser.cpp | 52 +++++++++++++++++++ 4 files changed, 70 insertions(+), 18 deletions(-) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp index 1d5e8a07a5..644a8bb75a 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp @@ -123,15 +123,7 @@ ST::string pfConsoleEngine::GetCmdSignature(const ST::string& name) pfConsoleCmd *cmd; pfConsoleParser parser(name); - auto [group, token] = parser.ParseGroupAndName(); - - if (!token) { - fErrorMsg = ST_LITERAL("Invalid command syntax"); - return {}; - } - - /// OK, so what we found wasn't a group. Which means we need a command... - cmd = group->FindCommandNoCase(*token); + cmd = parser.ParseCommand(); if (cmd == nullptr) { fErrorMsg = ST_LITERAL("Invalid syntax: command not found"); @@ -197,15 +189,7 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S bool valid = true; pfConsoleParser parser(line); - auto [group, token] = parser.ParseGroupAndName(); - - if (!token) { - fErrorMsg = ST_LITERAL("Invalid command syntax"); - return false; - } - - /// OK, so what we found wasn't a group. Which means we need a command next - cmd = group->FindCommandNoCase(*token); + cmd = parser.ParseCommand(); if (cmd == nullptr) { fErrorMsg = ST_LITERAL("Invalid syntax: command not found"); @@ -216,6 +200,7 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S /// tokenizing (with the new separators now, mind you) and turn them into /// params + std::optional token; for (numParams = 0; numParams < fMaxNumParams && (token = parser.fTokenizer.NextArgument()) && valid; numParams++ ) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp index 05b9715712..b4e4759762 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp @@ -133,3 +133,12 @@ std::pair> pfConsoleParser::ParseG return {group, token}; } + +pfConsoleCmd* pfConsoleParser::ParseCommand() +{ + auto [group, token] = ParseGroupAndName(); + if (!token) { + return nullptr; + } + return group->FindCommandNoCase(*token); +} diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h index c15d66a956..658af25edb 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h @@ -50,6 +50,7 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include #include +class pfConsoleCmd; class pfConsoleCmdGroup; class pfConsoleTokenizer @@ -89,6 +90,11 @@ class pfConsoleParser // Returns the found group and the first non-group token // (which may be an empty std::optional if the end of the line was reached). std::pair> ParseGroupAndName(); + + // Parse the command name part of the line. + // Returns the command corresponding to that name, + // or nullptr if no matching command was found. + pfConsoleCmd* ParseCommand(); }; #endif // _pfConsolePareser_h diff --git a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleParser.cpp b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleParser.cpp index 2bafebbf3c..750200f4a7 100644 --- a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleParser.cpp +++ b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleParser.cpp @@ -75,6 +75,10 @@ TEST(pfConsoleParser, ParseBaseCommand) EXPECT_EQ(group, pfConsoleCmdGroup::GetBaseGroup()); EXPECT_EQ(token, "TestBaseCmd"_st); EXPECT_EQ(parser.fTokenizer.fPos, string.end()); + + pfConsoleParser parser2(string); + auto cmd = parser2.ParseCommand(); + EXPECT_EQ(cmd, &conCmd_TestBaseCmd); } TEST(pfConsoleParser, ParseBaseCommandArgs) @@ -86,6 +90,10 @@ TEST(pfConsoleParser, ParseBaseCommandArgs) EXPECT_EQ(group, pfConsoleCmdGroup::GetBaseGroup()); EXPECT_EQ(token, "TestBaseCmd"_st); EXPECT_EQ(parser.fTokenizer.fPos, string.begin() + sizeof("TestBaseCmd ") - 1); + + pfConsoleParser parser2(string); + auto cmd = parser2.ParseCommand(); + EXPECT_EQ(cmd, &conCmd_TestBaseCmd); } // Top-level group @@ -99,6 +107,10 @@ TEST(pfConsoleParser, ParseBaseGroup) EXPECT_EQ(group, &conGroup_TestGroup); EXPECT_FALSE(token); EXPECT_EQ(parser.fTokenizer.fPos, string.end()); + + pfConsoleParser parser2(string); + auto cmd = parser2.ParseCommand(); + EXPECT_EQ(cmd, nullptr); } // Command inside top-level group @@ -112,6 +124,10 @@ TEST(pfConsoleParser, ParseSubCommand) EXPECT_EQ(group, &conGroup_TestGroup); EXPECT_EQ(token, "SubCmd"_st); EXPECT_EQ(parser.fTokenizer.fPos, string.end()); + + pfConsoleParser parser2(string); + auto cmd = parser2.ParseCommand(); + EXPECT_EQ(cmd, &conCmd_TestGroup_SubCmd); } TEST(pfConsoleParser, ParseSubCommandArgs) @@ -123,6 +139,10 @@ TEST(pfConsoleParser, ParseSubCommandArgs) EXPECT_EQ(group, &conGroup_TestGroup); EXPECT_EQ(token, "SubCmd"_st); EXPECT_EQ(parser.fTokenizer.fPos, string.begin() + sizeof("TestGroup.SubCmd ") - 1); + + pfConsoleParser parser2(string); + auto cmd = parser2.ParseCommand(); + EXPECT_EQ(cmd, &conCmd_TestGroup_SubCmd); } TEST(pfConsoleParser, ParseSubCommandSpace) @@ -134,6 +154,10 @@ TEST(pfConsoleParser, ParseSubCommandSpace) EXPECT_EQ(group, &conGroup_TestGroup); EXPECT_EQ(token, "SubCmd"_st); EXPECT_EQ(parser.fTokenizer.fPos, string.end()); + + pfConsoleParser parser2(string); + auto cmd = parser2.ParseCommand(); + EXPECT_EQ(cmd, &conCmd_TestGroup_SubCmd); } TEST(pfConsoleParser, ParseSubCommandSpaceArgs) @@ -145,6 +169,10 @@ TEST(pfConsoleParser, ParseSubCommandSpaceArgs) EXPECT_EQ(group, &conGroup_TestGroup); EXPECT_EQ(token, "SubCmd"_st); EXPECT_EQ(parser.fTokenizer.fPos, string.begin() + sizeof("TestGroup SubCmd ") - 1); + + pfConsoleParser parser2(string); + auto cmd = parser2.ParseCommand(); + EXPECT_EQ(cmd, &conCmd_TestGroup_SubCmd); } // Subgroup inside other group @@ -158,6 +186,10 @@ TEST(pfConsoleParser, ParseSubGroup) EXPECT_EQ(group, &conGroup_TestGroup_SubGroup); EXPECT_FALSE(token); EXPECT_EQ(parser.fTokenizer.fPos, string.end()); + + pfConsoleParser parser2(string); + auto cmd = parser2.ParseCommand(); + EXPECT_EQ(cmd, nullptr); } TEST(pfConsoleParser, ParseSubGroupSpace) @@ -169,6 +201,10 @@ TEST(pfConsoleParser, ParseSubGroupSpace) EXPECT_EQ(group, &conGroup_TestGroup_SubGroup); EXPECT_FALSE(token); EXPECT_EQ(parser.fTokenizer.fPos, string.end()); + + pfConsoleParser parser2(string); + auto cmd = parser2.ParseCommand(); + EXPECT_EQ(cmd, nullptr); } // Command inside subgroup @@ -182,6 +218,10 @@ TEST(pfConsoleParser, ParseSubSubCommand) EXPECT_EQ(group, &conGroup_TestGroup_SubGroup); EXPECT_EQ(token, "SubSubCmd"_st); EXPECT_EQ(parser.fTokenizer.fPos, string.end()); + + pfConsoleParser parser2(string); + auto cmd = parser2.ParseCommand(); + EXPECT_EQ(cmd, &conCmd_TestGroup_SubGroup_SubSubCmd); } TEST(pfConsoleParser, ParseSubSubCommandArgs) @@ -193,6 +233,10 @@ TEST(pfConsoleParser, ParseSubSubCommandArgs) EXPECT_EQ(group, &conGroup_TestGroup_SubGroup); EXPECT_EQ(token, "SubSubCmd"_st); EXPECT_EQ(parser.fTokenizer.fPos, string.begin() + sizeof("TestGroup.SubGroup.SubSubCmd ") - 1); + + pfConsoleParser parser2(string); + auto cmd = parser2.ParseCommand(); + EXPECT_EQ(cmd, &conCmd_TestGroup_SubGroup_SubSubCmd); } TEST(pfConsoleParser, ParseSubSubCommandSpaces) @@ -204,6 +248,10 @@ TEST(pfConsoleParser, ParseSubSubCommandSpaces) EXPECT_EQ(group, &conGroup_TestGroup_SubGroup); EXPECT_EQ(token, "SubSubCmd"_st); EXPECT_EQ(parser.fTokenizer.fPos, string.end()); + + pfConsoleParser parser2(string); + auto cmd = parser2.ParseCommand(); + EXPECT_EQ(cmd, &conCmd_TestGroup_SubGroup_SubSubCmd); } TEST(pfConsoleParser, ParseSubSubCommandSpacesArgs) @@ -215,4 +263,8 @@ TEST(pfConsoleParser, ParseSubSubCommandSpacesArgs) EXPECT_EQ(group, &conGroup_TestGroup_SubGroup); EXPECT_EQ(token, "SubSubCmd"_st); EXPECT_EQ(parser.fTokenizer.fPos, string.begin() + sizeof("TestGroup SubGroup SubSubCmd ") - 1); + + pfConsoleParser parser2(string); + auto cmd = parser2.ParseCommand(); + EXPECT_EQ(cmd, &conCmd_TestGroup_SubGroup_SubSubCmd); } From 5ecc63f1318624c6064b446b24659375f1219b0d Mon Sep 17 00:00:00 2001 From: dgelessus Date: Tue, 1 Aug 2023 00:12:53 +0200 Subject: [PATCH 13/17] Refactor console command argument parsing --- .../pfConsoleCore/pfConsoleEngine.cpp | 22 +++++----- .../pfConsoleCore/pfConsoleParser.cpp | 13 ++++++ .../pfConsoleCore/pfConsoleParser.h | 8 ++++ .../test_pfConsoleParser.cpp | 40 +++++++++++++++++++ 4 files changed, 72 insertions(+), 11 deletions(-) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp index 644a8bb75a..eaf6a1b378 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp @@ -200,19 +200,25 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S /// tokenizing (with the new separators now, mind you) and turn them into /// params - std::optional token; + auto argTokens = parser.ParseArguments(); + if (!argTokens) { + fErrorMsg = ST::format("Invalid syntax: {}", parser.GetErrorMsg()); + return false; + } + for (numParams = 0; numParams < fMaxNumParams - && (token = parser.fTokenizer.NextArgument()) + && numParams < argTokens->size() && valid; numParams++ ) { // Special case for context variables--if we're specifying one, we want to just grab // the value of it and return that instead valid = false; - if (token->starts_with("$")) { + ST::string argString = (*argTokens)[numParams]; + if (argString.starts_with("$")) { pfConsoleContext &context = pfConsoleContext::GetRootContext(); // Potential variable, see if we can find it - hsSsize_t idx = context.FindVar(token->substr(1)); + hsSsize_t idx = context.FindVar(argString.substr(1)); if( idx == -1 ) { fErrorMsg = ST_LITERAL("Invalid console variable name"); @@ -225,13 +231,7 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S } if( !valid ) - valid = IConvertToParam(cmd->GetSigEntry(numParams), *token, ¶mArray[numParams]); - } - - ST::string errorMsg = parser.fTokenizer.fErrorMsg; - if (!errorMsg.empty()) { - fErrorMsg = ST::format("Invalid syntax: {}", errorMsg); - return false; + valid = IConvertToParam(cmd->GetSigEntry(numParams), argString, ¶mArray[numParams]); } for( i = numParams; i < fMaxNumParams + 1; i++ ) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp index b4e4759762..be46c1fc53 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp @@ -142,3 +142,16 @@ pfConsoleCmd* pfConsoleParser::ParseCommand() } return group->FindCommandNoCase(*token); } + +std::optional> pfConsoleParser::ParseArguments() +{ + std::vector args; + while (auto token = fTokenizer.NextArgument()) { + args.emplace_back(std::move(*token)); + } + if (!fTokenizer.fErrorMsg.empty()) { + // Parse error in argument + return {}; + } + return args; +} diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h index 658af25edb..33acebbea0 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h @@ -84,6 +84,8 @@ class pfConsoleParser pfConsoleParser(ST::string::const_iterator begin, ST::string::const_iterator end) : fTokenizer(begin, end) {} pfConsoleParser(const ST::string& line) : pfConsoleParser(line.begin(), line.end()) {} + ST::string GetErrorMsg() const { return fTokenizer.fErrorMsg; } + // Parse the command name part of the line as far as possible. // This consumes name part tokens and uses them to look up a command group // until a token is encountered that isn't a known group name. @@ -95,6 +97,12 @@ class pfConsoleParser // Returns the command corresponding to that name, // or nullptr if no matching command was found. pfConsoleCmd* ParseCommand(); + + // Parse the remainder of the line as command arguments. + // On success, returns the parsed arguments with any surrounding quotes removed. + // If any of the arguments couldn't be parsed, + // returns an empty std::optional (call GetErrorMsg for an error message). + std::optional> ParseArguments(); }; #endif // _pfConsolePareser_h diff --git a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleParser.cpp b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleParser.cpp index 750200f4a7..e1826eaa75 100644 --- a/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleParser.cpp +++ b/Sources/Tests/FeatureTests/pfConsoleCoreTest/test_pfConsoleParser.cpp @@ -79,6 +79,9 @@ TEST(pfConsoleParser, ParseBaseCommand) pfConsoleParser parser2(string); auto cmd = parser2.ParseCommand(); EXPECT_EQ(cmd, &conCmd_TestBaseCmd); + auto args = parser2.ParseArguments(); + EXPECT_TRUE(args); + EXPECT_TRUE(args->empty()); } TEST(pfConsoleParser, ParseBaseCommandArgs) @@ -94,6 +97,11 @@ TEST(pfConsoleParser, ParseBaseCommandArgs) pfConsoleParser parser2(string); auto cmd = parser2.ParseCommand(); EXPECT_EQ(cmd, &conCmd_TestBaseCmd); + auto args = parser2.ParseArguments(); + EXPECT_TRUE(args); + EXPECT_EQ(args->size(), 2); + EXPECT_EQ((*args)[0], "arg1"_st); + EXPECT_EQ((*args)[1], "arg2"_st); } // Top-level group @@ -128,6 +136,9 @@ TEST(pfConsoleParser, ParseSubCommand) pfConsoleParser parser2(string); auto cmd = parser2.ParseCommand(); EXPECT_EQ(cmd, &conCmd_TestGroup_SubCmd); + auto args = parser2.ParseArguments(); + EXPECT_TRUE(args); + EXPECT_TRUE(args->empty()); } TEST(pfConsoleParser, ParseSubCommandArgs) @@ -143,6 +154,11 @@ TEST(pfConsoleParser, ParseSubCommandArgs) pfConsoleParser parser2(string); auto cmd = parser2.ParseCommand(); EXPECT_EQ(cmd, &conCmd_TestGroup_SubCmd); + auto args = parser2.ParseArguments(); + EXPECT_TRUE(args); + EXPECT_EQ(args->size(), 2); + EXPECT_EQ((*args)[0], "arg1"_st); + EXPECT_EQ((*args)[1], "arg2"_st); } TEST(pfConsoleParser, ParseSubCommandSpace) @@ -158,6 +174,9 @@ TEST(pfConsoleParser, ParseSubCommandSpace) pfConsoleParser parser2(string); auto cmd = parser2.ParseCommand(); EXPECT_EQ(cmd, &conCmd_TestGroup_SubCmd); + auto args = parser2.ParseArguments(); + EXPECT_TRUE(args); + EXPECT_TRUE(args->empty()); } TEST(pfConsoleParser, ParseSubCommandSpaceArgs) @@ -173,6 +192,11 @@ TEST(pfConsoleParser, ParseSubCommandSpaceArgs) pfConsoleParser parser2(string); auto cmd = parser2.ParseCommand(); EXPECT_EQ(cmd, &conCmd_TestGroup_SubCmd); + auto args = parser2.ParseArguments(); + EXPECT_TRUE(args); + EXPECT_EQ(args->size(), 2); + EXPECT_EQ((*args)[0], "arg1"_st); + EXPECT_EQ((*args)[1], "arg2"_st); } // Subgroup inside other group @@ -222,6 +246,9 @@ TEST(pfConsoleParser, ParseSubSubCommand) pfConsoleParser parser2(string); auto cmd = parser2.ParseCommand(); EXPECT_EQ(cmd, &conCmd_TestGroup_SubGroup_SubSubCmd); + auto args = parser2.ParseArguments(); + EXPECT_TRUE(args); + EXPECT_TRUE(args->empty()); } TEST(pfConsoleParser, ParseSubSubCommandArgs) @@ -237,6 +264,11 @@ TEST(pfConsoleParser, ParseSubSubCommandArgs) pfConsoleParser parser2(string); auto cmd = parser2.ParseCommand(); EXPECT_EQ(cmd, &conCmd_TestGroup_SubGroup_SubSubCmd); + auto args = parser2.ParseArguments(); + EXPECT_TRUE(args); + EXPECT_EQ(args->size(), 2); + EXPECT_EQ((*args)[0], "arg1"_st); + EXPECT_EQ((*args)[1], "arg2"_st); } TEST(pfConsoleParser, ParseSubSubCommandSpaces) @@ -252,6 +284,9 @@ TEST(pfConsoleParser, ParseSubSubCommandSpaces) pfConsoleParser parser2(string); auto cmd = parser2.ParseCommand(); EXPECT_EQ(cmd, &conCmd_TestGroup_SubGroup_SubSubCmd); + auto args = parser2.ParseArguments(); + EXPECT_TRUE(args); + EXPECT_TRUE(args->empty()); } TEST(pfConsoleParser, ParseSubSubCommandSpacesArgs) @@ -267,4 +302,9 @@ TEST(pfConsoleParser, ParseSubSubCommandSpacesArgs) pfConsoleParser parser2(string); auto cmd = parser2.ParseCommand(); EXPECT_EQ(cmd, &conCmd_TestGroup_SubGroup_SubSubCmd); + auto args = parser2.ParseArguments(); + EXPECT_TRUE(args); + EXPECT_EQ(args->size(), 2); + EXPECT_EQ((*args)[0], "arg1"_st); + EXPECT_EQ((*args)[1], "arg2"_st); } From 92034d9c09c2303bf2a7fce5a177553d40d6fc08 Mon Sep 17 00:00:00 2001 From: dgelessus Date: Thu, 3 Aug 2023 01:07:28 +0200 Subject: [PATCH 14/17] Refactor console argument type checking and variable handling --- .../pfConsoleCore/pfConsoleEngine.cpp | 74 ++++++++++--------- .../pfConsoleCore/pfConsoleEngine.h | 3 + 2 files changed, 41 insertions(+), 36 deletions(-) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp index eaf6a1b378..1c08dd7138 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp @@ -53,7 +53,6 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include #include #include -#include #include "plFile/plEncryptedStream.h" @@ -184,9 +183,7 @@ bool pfConsoleEngine::ExecuteFile(const plFileName &fileName) bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const ST::string&)) { pfConsoleCmd *cmd; - int32_t numParams, i; pfConsoleCmdParam paramArray[ fMaxNumParams + 1 ]; - bool valid = true; pfConsoleParser parser(line); cmd = parser.ParseCommand(); @@ -206,40 +203,12 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S return false; } - for (numParams = 0; numParams < fMaxNumParams - && numParams < argTokens->size() - && valid; numParams++ ) - { - // Special case for context variables--if we're specifying one, we want to just grab - // the value of it and return that instead - valid = false; - ST::string argString = (*argTokens)[numParams]; - if (argString.starts_with("$")) { - pfConsoleContext &context = pfConsoleContext::GetRootContext(); + hsSsize_t numParams = IResolveParams(cmd, std::move(*argTokens), paramArray); - // Potential variable, see if we can find it - hsSsize_t idx = context.FindVar(argString.substr(1)); - if( idx == -1 ) - { - fErrorMsg = ST_LITERAL("Invalid console variable name"); - } - else - { - paramArray[ numParams ] = context.GetVarValue( idx ); - valid = true; - } - } - - if( !valid ) - valid = IConvertToParam(cmd->GetSigEntry(numParams), argString, ¶mArray[numParams]); - } - - for( i = numParams; i < fMaxNumParams + 1; i++ ) - paramArray[ i ].SetNone(); - - if (!valid || (cmd->GetSigEntry(numParams) != pfConsoleCmd::kAny && - cmd->GetSigEntry(numParams) != pfConsoleCmd::kNone)) - { + if (numParams == -1 || ( + cmd->GetSigEntry(numParams) != pfConsoleCmd::kAny + && cmd->GetSigEntry(numParams) != pfConsoleCmd::kNone + )) { // Print help string and return fErrorMsg.clear(); // Printed on next line PrintFn(ST_LITERAL("Invalid parameters to command")); @@ -305,6 +274,39 @@ bool pfConsoleEngine::IConvertToParam(uint8_t type, ST::string string, pfConsole return true; } +hsSsize_t pfConsoleEngine::IResolveParams(pfConsoleCmd* cmd, std::vector argTokens, pfConsoleCmdParam* paramArray) +{ + size_t numParams; + for (numParams = 0; numParams < fMaxNumParams && numParams < argTokens.size(); numParams++) { + // Special case for context variables--if we're specifying one, + // we want to just grab the value of it and return that instead + ST::string& argString = argTokens[numParams]; + if (argString.starts_with("$")) { + pfConsoleContext& context = pfConsoleContext::GetRootContext(); + + // Potential variable, see if we can find it + hsSsize_t idx = context.FindVar(argString.substr(1)); + if (idx == -1) { + fErrorMsg = ST_LITERAL("Invalid console variable name"); + } else { + paramArray[numParams] = context.GetVarValue(idx); + continue; + } + } + + bool valid = IConvertToParam(cmd->GetSigEntry(numParams), std::move(argString), ¶mArray[numParams]); + if (!valid) { + return -1; + } + } + + for (size_t i = numParams; i < fMaxNumParams + 1; i++) { + paramArray[i].SetNone(); + } + + return numParams; +} + //// FindPartialCmd ////////////////////////////////////////////////////////// // Given a string which is the beginning of a console command, // returns the best match of command (or group) for that string. diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h index 0b3c25421b..bcaa89c9c6 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h @@ -58,11 +58,13 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include "HeadSpin.h" #include +#include class plFileName; //// pfConsoleEngine Class Definition //////////////////////////////////////// +class pfConsoleCmd; class pfConsoleCmdParam; class pfConsoleCmdGroup; class pfConsoleEngine @@ -72,6 +74,7 @@ class pfConsoleEngine static const int32_t fMaxNumParams; bool IConvertToParam(uint8_t type, ST::string string, pfConsoleCmdParam *param); + hsSsize_t IResolveParams(pfConsoleCmd* cmd, std::vector argTokens, pfConsoleCmdParam* paramArray); ST::string fErrorMsg; ST::string fLastErrorLine; From ff88e4b7ea98e12e37e4a84ad678bfc8300400a9 Mon Sep 17 00:00:00 2001 From: dgelessus Date: Thu, 3 Aug 2023 22:20:50 +0200 Subject: [PATCH 15/17] Replace fixed-size console paramsArray with vector --- .../pfConsoleCore/pfConsoleEngine.cpp | 61 ++++++++----------- .../pfConsoleCore/pfConsoleEngine.h | 8 +-- 2 files changed, 30 insertions(+), 39 deletions(-) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp index 1c08dd7138..233f8327b1 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp @@ -57,9 +57,6 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include "plFile/plEncryptedStream.h" -const int32_t pfConsoleEngine::fMaxNumParams = 16; - - //// Constructor & Destructor //////////////////////////////////////////////// pfConsoleEngine::pfConsoleEngine() @@ -183,7 +180,6 @@ bool pfConsoleEngine::ExecuteFile(const plFileName &fileName) bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const ST::string&)) { pfConsoleCmd *cmd; - pfConsoleCmdParam paramArray[ fMaxNumParams + 1 ]; pfConsoleParser parser(line); cmd = parser.ParseCommand(); @@ -203,11 +199,11 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S return false; } - hsSsize_t numParams = IResolveParams(cmd, std::move(*argTokens), paramArray); + auto params = IResolveParams(cmd, std::move(*argTokens)); - if (numParams == -1 || ( - cmd->GetSigEntry(numParams) != pfConsoleCmd::kAny - && cmd->GetSigEntry(numParams) != pfConsoleCmd::kNone + if (!params || ( + cmd->GetSigEntry(params->size()) != pfConsoleCmd::kAny + && cmd->GetSigEntry(params->size()) != pfConsoleCmd::kNone )) { // Print help string and return fErrorMsg.clear(); // Printed on next line @@ -217,7 +213,7 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S } /// Execute it and return - cmd->Execute( numParams, paramArray, PrintFn ); + cmd->Execute(params->size(), params->data(), PrintFn); return true; } @@ -225,62 +221,62 @@ bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const S // Converts a null-terminated string representing a parameter to a // pfConsoleCmdParam argument. -bool pfConsoleEngine::IConvertToParam(uint8_t type, ST::string string, pfConsoleCmdParam *param) +std::optional pfConsoleEngine::IConvertToParam(uint8_t type, ST::string string) { if( type == pfConsoleCmd::kNone ) - return false; + return {}; + pfConsoleCmdParam param; if( type == pfConsoleCmd::kAny ) { /// Want "any" - param->SetAny(std::move(string)); + param.SetAny(std::move(string)); } else if( type == pfConsoleCmd::kString ) { /// Want just a string - param->SetString(std::move(string)); + param.SetString(std::move(string)); } else if( type == pfConsoleCmd::kFloat ) { ST::conversion_result res; float value = string.to_float(res); if (!res.ok() || !res.full_match()) { - return false; + return {}; } - param->SetFloat(value); + param.SetFloat(value); } else if( type == pfConsoleCmd::kInt ) { ST::conversion_result res; int value = string.to_int(res, 10); if (!res.ok() || !res.full_match()) { - return false; + return {}; } - param->SetInt(value); + param.SetInt(value); } else if( type == pfConsoleCmd::kBool ) { if (string.compare_i("t") == 0) { - param->SetBool(true); + param.SetBool(true); } else if (string.compare_i("f") == 0) { - param->SetBool(false); + param.SetBool(false); } else { - param->SetBool(string.to_bool()); + param.SetBool(string.to_bool()); } } - return true; + return param; } -hsSsize_t pfConsoleEngine::IResolveParams(pfConsoleCmd* cmd, std::vector argTokens, pfConsoleCmdParam* paramArray) +std::optional> pfConsoleEngine::IResolveParams(pfConsoleCmd* cmd, std::vector argTokens) { - size_t numParams; - for (numParams = 0; numParams < fMaxNumParams && numParams < argTokens.size(); numParams++) { + std::vector params; + for (ST::string& argString : argTokens) { // Special case for context variables--if we're specifying one, // we want to just grab the value of it and return that instead - ST::string& argString = argTokens[numParams]; if (argString.starts_with("$")) { pfConsoleContext& context = pfConsoleContext::GetRootContext(); @@ -289,22 +285,19 @@ hsSsize_t pfConsoleEngine::IResolveParams(pfConsoleCmd* cmd, std::vectorGetSigEntry(numParams), std::move(argString), ¶mArray[numParams]); - if (!valid) { - return -1; + auto param = IConvertToParam(cmd->GetSigEntry(params.size()), std::move(argString)); + if (!param) { + return {}; } + params.emplace_back(std::move(*param)); } - for (size_t i = numParams; i < fMaxNumParams + 1; i++) { - paramArray[i].SetNone(); - } - - return numParams; + return params; } //// FindPartialCmd ////////////////////////////////////////////////////////// diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h index bcaa89c9c6..969f2f2af3 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.h @@ -57,6 +57,7 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include "HeadSpin.h" +#include #include #include @@ -70,11 +71,8 @@ class pfConsoleCmdGroup; class pfConsoleEngine { private: - - static const int32_t fMaxNumParams; - - bool IConvertToParam(uint8_t type, ST::string string, pfConsoleCmdParam *param); - hsSsize_t IResolveParams(pfConsoleCmd* cmd, std::vector argTokens, pfConsoleCmdParam* paramArray); + std::optional IConvertToParam(uint8_t type, ST::string string); + std::optional> IResolveParams(pfConsoleCmd* cmd, std::vector argTokens); ST::string fErrorMsg; ST::string fLastErrorLine; From 9d5e3a1d063734bf5de2a5d751f0fc14df7a8450 Mon Sep 17 00:00:00 2001 From: dgelessus Date: Fri, 11 Aug 2023 00:53:46 +0200 Subject: [PATCH 16/17] Fix the last C-style local variables in pfConsoleEngine --- .../pfConsoleCore/pfConsoleEngine.cpp | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp index 233f8327b1..9018ac112f 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleEngine.cpp @@ -71,8 +71,6 @@ pfConsoleEngine::~pfConsoleEngine() bool pfConsoleEngine::PrintCmdHelp(const ST::string& name, void (*PrintFn)(const ST::string&)) { - pfConsoleCmd *cmd; - pfConsoleParser parser(name); auto [group, token] = parser.ParseGroupAndName(); @@ -88,7 +86,7 @@ bool pfConsoleEngine::PrintCmdHelp(const ST::string& name, void (*PrintFn)(const PrintFn(ST::format(" {}", subGrp->GetName())); } PrintFn(ST_LITERAL(" Commands:")); - for (cmd = group->GetFirstCommand(); cmd != nullptr; cmd = cmd->GetNext()) + for (pfConsoleCmd* cmd = group->GetFirstCommand(); cmd != nullptr; cmd = cmd->GetNext()) { PrintFn(ST::format(" {}: {}", cmd->GetName(), cmd->GetHelp().before_first('\n'))); } @@ -97,7 +95,7 @@ bool pfConsoleEngine::PrintCmdHelp(const ST::string& name, void (*PrintFn)(const } /// OK, so what we found wasn't a group. Which means we need a command... - cmd = group->FindCommandNoCase(*token); + pfConsoleCmd* cmd = group->FindCommandNoCase(*token); if (cmd == nullptr) { fErrorMsg = ST_LITERAL("Invalid syntax: command not found"); @@ -116,10 +114,8 @@ bool pfConsoleEngine::PrintCmdHelp(const ST::string& name, void (*PrintFn)(const ST::string pfConsoleEngine::GetCmdSignature(const ST::string& name) { - pfConsoleCmd *cmd; - pfConsoleParser parser(name); - cmd = parser.ParseCommand(); + pfConsoleCmd* cmd = parser.ParseCommand(); if (cmd == nullptr) { fErrorMsg = ST_LITERAL("Invalid syntax: command not found"); @@ -140,8 +136,6 @@ void DummyPrintFn(const ST::string& line) bool pfConsoleEngine::ExecuteFile(const plFileName &fileName) { - int line; - std::unique_ptr stream = plEncryptedStream::OpenEncryptedFile(fileName); if( !stream ) @@ -156,7 +150,7 @@ bool pfConsoleEngine::ExecuteFile(const plFileName &fileName) } ST::string string; - for (line = 1; stream->ReadLn(string); line++) + for (int line = 1; stream->ReadLn(string); line++) { fLastErrorLine = string; @@ -179,10 +173,8 @@ bool pfConsoleEngine::ExecuteFile(const plFileName &fileName) bool pfConsoleEngine::RunCommand(const ST::string& line, void (*PrintFn)(const ST::string&)) { - pfConsoleCmd *cmd; - pfConsoleParser parser(line); - cmd = parser.ParseCommand(); + pfConsoleCmd* cmd = parser.ParseCommand(); if (cmd == nullptr) { fErrorMsg = ST_LITERAL("Invalid syntax: command not found"); From 1ccefa375e029795f6fe61aa75d80d437fe9af8a Mon Sep 17 00:00:00 2001 From: dgelessus Date: Sat, 16 Sep 2023 23:18:04 +0200 Subject: [PATCH 17/17] Use std::tuple instead of std::pair for pfConsoleParser returns --- Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp | 4 +++- Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp index be46c1fc53..8c1a0957ea 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.cpp @@ -42,6 +42,8 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include "pfConsoleParser.h" +#include + #include "pfConsoleCmd.h" static const char kTokenSeparators[] = " =\r\n\t,"; @@ -117,7 +119,7 @@ std::optional pfConsoleTokenizer::NextArgument() return begin; } -std::pair> pfConsoleParser::ParseGroupAndName() +std::tuple> pfConsoleParser::ParseGroupAndName() { pfConsoleCmdGroup* group = pfConsoleCmdGroup::GetBaseGroup(); auto token = fTokenizer.NextNamePart(); diff --git a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h index 33acebbea0..8a8680ee20 100644 --- a/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h +++ b/Sources/Plasma/FeatureLib/pfConsoleCore/pfConsoleParser.h @@ -47,7 +47,7 @@ You can contact Cyan Worlds, Inc. by email legal@cyan.com #include #include -#include +#include #include class pfConsoleCmd; @@ -91,7 +91,7 @@ class pfConsoleParser // until a token is encountered that isn't a known group name. // Returns the found group and the first non-group token // (which may be an empty std::optional if the end of the line was reached). - std::pair> ParseGroupAndName(); + std::tuple> ParseGroupAndName(); // Parse the command name part of the line. // Returns the command corresponding to that name,