From 97e743b33dfa76194f7a9915adef88dbd5385516 Mon Sep 17 00:00:00 2001 From: MarkusRannare Date: Sun, 17 Feb 2013 16:25:40 +0100 Subject: [PATCH] * Start on TokenGenerator * Moved out testcases to own headers --- Dev/src/fry_script/fry_script.vcxproj | 3 + Dev/src/fry_script/fry_script.vcxproj.filters | 9 ++ Dev/src/fry_script/token_generator.cpp | 74 ++++++++++++ Dev/src/fry_script/token_generator.h | 22 ++++ Dev/src/fry_script/token_generator_types.h | 31 +++++ .../fry_script_test/fry_script_test.vcxproj | 4 + .../fry_script_test.vcxproj.filters | 8 ++ Dev/src/fry_script_test/interpretor_tests.h | 105 +++++++++++++++++ Dev/src/fry_script_test/main.cpp | 107 +----------------- .../fry_script_test/token_generator_tests.h | 96 ++++++++++++++++ 10 files changed, 354 insertions(+), 105 deletions(-) create mode 100644 Dev/src/fry_script/token_generator.cpp create mode 100644 Dev/src/fry_script/token_generator.h create mode 100644 Dev/src/fry_script/token_generator_types.h create mode 100644 Dev/src/fry_script_test/interpretor_tests.h create mode 100644 Dev/src/fry_script_test/token_generator_tests.h diff --git a/Dev/src/fry_script/fry_script.vcxproj b/Dev/src/fry_script/fry_script.vcxproj index 7fe753f..bdcb8fa 100644 --- a/Dev/src/fry_script/fry_script.vcxproj +++ b/Dev/src/fry_script/fry_script.vcxproj @@ -88,9 +88,12 @@ + + + diff --git a/Dev/src/fry_script/fry_script.vcxproj.filters b/Dev/src/fry_script/fry_script.vcxproj.filters index 6b3d68f..7567f14 100644 --- a/Dev/src/fry_script/fry_script.vcxproj.filters +++ b/Dev/src/fry_script/fry_script.vcxproj.filters @@ -33,10 +33,19 @@ Header Files + + Header Files + + + Header Files + Source Files + + Source Files + \ No newline at end of file diff --git a/Dev/src/fry_script/token_generator.cpp b/Dev/src/fry_script/token_generator.cpp new file mode 100644 index 0000000..989990c --- /dev/null +++ b/Dev/src/fry_script/token_generator.cpp @@ -0,0 +1,74 @@ +#include "token_generator.h" +#include +#include + +using namespace foundation; + +static const char CharsToIgnore[] = +{ + // Whitespaces + ' ', + '\t', + '\n', + '\r' +}; + +#include + +namespace fry_script +{ + namespace token_generator + { + bool GenerateTokens( const char* Code, Array& out_Tokens ) + { + GeneratedToken Tok; + size_t LastFoundIdx = 0; + + Tok.Location = Code; + Tok.Length = 0; + + size_t Idx = 0; + for( ; Idx < strlen( Code ); ++Idx ) + { + if( Code[Idx] == ' ' ) + { + Tok.Length = Idx - LastFoundIdx; + + array::push_back( out_Tokens, Tok ); + + LastFoundIdx = Idx; + Tok.Location = &Code[Idx+1]; + Tok.Length = 0; + } + } + + Tok.Length = Idx - LastFoundIdx; + if( Tok.Length > 0 ) + { + array::push_back( out_Tokens, Tok ); + } + + return true; + } + + bool GetNextToken( TokenGenerator* Generator, Array& Tokens, const char** out_NextToken ) + { + size_t CurrentIndex = Generator->CurrentIndex++; + if( CurrentIndex >= array::size( Tokens ) ) + { + Generator->CurrentIndex = -1; + memory::mem_zero( Generator->TempToken, sizeof( Generator->TempToken ) ); + + return false; + } + + const GeneratedToken& CurrentToken = Tokens[CurrentIndex]; + + memcpy_s( Generator->TempToken, sizeof( Generator->TempToken ), CurrentToken.Location, CurrentToken.Length ); + Generator->TempToken[CurrentToken.Length] = '\0'; + *out_NextToken = Generator->TempToken; + + return true; + } + } +} \ No newline at end of file diff --git a/Dev/src/fry_script/token_generator.h b/Dev/src/fry_script/token_generator.h new file mode 100644 index 0000000..9d01ea9 --- /dev/null +++ b/Dev/src/fry_script/token_generator.h @@ -0,0 +1,22 @@ +#ifndef FRY_SCRIPT_TOKEN_GENERATOR_H +#define FRY_SCRIPT_TOKEN_GENERATOR_H + +#include +#include "token_generator_types.h" + +namespace fry_script +{ + namespace token_generator + { + /// Generates tokens from code + /// \param Code - the code to generator tokens from + /// \param Allocator - the allocator that will allocate the tokens in out_Tokens + /// \param out_Tokens - the tokes generated from Code + bool GenerateTokens( const char* Code, foundation::Array& out_Tokens ); + + /// Copies a single token + bool GetNextToken( TokenGenerator* Generator, foundation::Array& Tokens, const char** out_NextToken ); + } +} + +#endif \ No newline at end of file diff --git a/Dev/src/fry_script/token_generator_types.h b/Dev/src/fry_script/token_generator_types.h new file mode 100644 index 0000000..8c1ea66 --- /dev/null +++ b/Dev/src/fry_script/token_generator_types.h @@ -0,0 +1,31 @@ +#ifndef FRY_SCRIPT_TOKEN_GENERATOR_TYPES +#define FRY_SCRIPT_TOKEN_GENERATOR_TYPES + +namespace fry_script +{ + /// Largest size of a allowed token + const s32 MaxTokenSize = 255; + + /// Struct that describes the tokens generated from GenerateTokens + /// It contains a pointer into the char* Code that is sent into the + /// GenerateToken function + struct GeneratedToken + { + const char* Location; + u32 Length; + }; + + /// The state of a TokenGenerator (for multithread support) + struct TokenGenerator + { + TokenGenerator() : + CurrentIndex( 0 ) + { + } + + int CurrentIndex; + char TempToken[MaxTokenSize]; + }; +} + +#endif \ No newline at end of file diff --git a/Dev/src/fry_script_test/fry_script_test.vcxproj b/Dev/src/fry_script_test/fry_script_test.vcxproj index 9cd4f18..a1675bc 100644 --- a/Dev/src/fry_script_test/fry_script_test.vcxproj +++ b/Dev/src/fry_script_test/fry_script_test.vcxproj @@ -13,6 +13,10 @@ + + + + {97E8EAAA-7115-406C-9F98-DD88CAB931F2} Win32Proj diff --git a/Dev/src/fry_script_test/fry_script_test.vcxproj.filters b/Dev/src/fry_script_test/fry_script_test.vcxproj.filters index 2a14256..6e8613e 100644 --- a/Dev/src/fry_script_test/fry_script_test.vcxproj.filters +++ b/Dev/src/fry_script_test/fry_script_test.vcxproj.filters @@ -19,4 +19,12 @@ Source Files + + + Header Files + + + Header Files + + \ No newline at end of file diff --git a/Dev/src/fry_script_test/interpretor_tests.h b/Dev/src/fry_script_test/interpretor_tests.h new file mode 100644 index 0000000..9d49083 --- /dev/null +++ b/Dev/src/fry_script_test/interpretor_tests.h @@ -0,0 +1,105 @@ +#include "UnitTest++.h" + +#include +#include +#include + +using namespace foundation; +using namespace fry_core; +using namespace fry_script; + +struct InterpreterCleanSetup +{ + InterpreterCleanSetup() : + Interp( memory_globals::default_allocator() ), + Bytecode( memory_globals::default_allocator() ), + Stack( Interp._Stack ) + { + } + + ~InterpreterCleanSetup() + { + } + + #pragma warning( disable: 4100 ) + InterpreterCleanSetup& operator=( const InterpreterCleanSetup& Other ) + { + // Should never be here, but can't use CHECK as FRY_CORE::CHECK conflicts with C++TEST::CHECK + return *this; + } + #pragma warning( default: 4100 ) + + Interpreter Interp; + Array Bytecode; + // Helper to prevent us from typing Interp._Stack all the time + Array& Stack; +}; + +SUITE( InterpreterPushI ) +{ + TEST_FIXTURE( InterpreterCleanSetup, PushI0 ) + { + CHECK_EQUAL( (unsigned)0, array::size( Stack ) ); + + ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI ); + ADD_CONSTANT( Bytecode, s32, 0 ); + + bool Result = interpreter::Run( Interp, array::begin( Bytecode ), array::size( Bytecode ) ); + + CHECK( Result ); + CHECK_EQUAL( (unsigned)4, array::size( Stack ) ); + GET_ITEM( Stack, s32, PushResult ); + CHECK_EQUAL( 0, PushResult ); + } + + TEST_FIXTURE( InterpreterCleanSetup, PushI666 ) + { + CHECK_EQUAL( (unsigned)0, array::size( Stack ) ); + + ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI ); + ADD_CONSTANT( Bytecode, s32, 666 ); + + bool Result = interpreter::Run( Interp, array::begin( Bytecode ), array::size( Bytecode ) ); + + CHECK( Result ); + CHECK_EQUAL( (unsigned)4, array::size( Stack ) ); + GET_ITEM( Stack, s32, PushResult ); + CHECK_EQUAL( 666, PushResult ); + } +} + +SUITE( InterpreterAddI ) +{ + TEST_FIXTURE( InterpreterCleanSetup, AddI00 ) + { + ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI ); + ADD_CONSTANT( Bytecode, s32, 0 ); + ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI ); + ADD_CONSTANT( Bytecode, s32, 0 ); + ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_AddI ); + + bool Result = interpreter::Run( Interp, array::begin( Bytecode ), array::size( Bytecode ) ); + + CHECK( Result ); + CHECK_EQUAL( (unsigned)4, array::size( Stack ) ); + GET_ITEM( Stack, s32, ResultValue ); + CHECK_EQUAL( 0, ResultValue ); + } + + TEST_FIXTURE( InterpreterCleanSetup, AddI42666 ) + { + ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI ); + ADD_CONSTANT( Bytecode, s32, 42 ); + ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI ); + ADD_CONSTANT( Bytecode, s32, 666 ); + ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_AddI ); + + bool Result = interpreter::Run( Interp, array::begin( Bytecode ), array::size( Bytecode ) ); + + CHECK( Result ); + CHECK_EQUAL( (unsigned)4, array::size( Stack ) ); + + GET_ITEM( Stack, s32, ResultValue ); + CHECK_EQUAL( 708, ResultValue ); + } +} \ No newline at end of file diff --git a/Dev/src/fry_script_test/main.cpp b/Dev/src/fry_script_test/main.cpp index 8ff65f9..0ea8853 100644 --- a/Dev/src/fry_script_test/main.cpp +++ b/Dev/src/fry_script_test/main.cpp @@ -1,108 +1,5 @@ -#include "UnitTest++.h" - -#include -#include -#include - -using namespace foundation; -using namespace fry_core; -using namespace fry_script; - -struct InterpreterCleanSetup -{ - InterpreterCleanSetup() : - Interp( memory_globals::default_allocator() ), - Bytecode( memory_globals::default_allocator() ), - Stack( Interp._Stack ) - { - } - - ~InterpreterCleanSetup() - { - } - -#pragma warning( disable: 4100 ) - InterpreterCleanSetup& operator=( const InterpreterCleanSetup& Other ) - { - // Should never be here, but can't use CHECK as FRY_CORE::CHECK conflicts with C++TEST::CHECK - return *this; - } -#pragma warning( default: 4100 ) - - Interpreter Interp; - Array Bytecode; - // Helper to prevent us from typing Interp._Stack all the time - Array& Stack; -}; - -SUITE( InterpreterPushI ) -{ - TEST_FIXTURE( InterpreterCleanSetup, PushI0 ) - { - CHECK_EQUAL( (unsigned)0, array::size( Stack ) ); - - ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI ); - ADD_CONSTANT( Bytecode, s32, 0 ); - - bool Result = interpreter::Run( Interp, array::begin( Bytecode ), array::size( Bytecode ) ); - - CHECK( Result ); - CHECK_EQUAL( (unsigned)4, array::size( Stack ) ); - GET_ITEM( Stack, s32, PushResult ); - CHECK_EQUAL( 0, PushResult ); - } - - TEST_FIXTURE( InterpreterCleanSetup, PushI666 ) - { - CHECK_EQUAL( (unsigned)0, array::size( Stack ) ); - - ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI ); - ADD_CONSTANT( Bytecode, s32, 666 ); - - bool Result = interpreter::Run( Interp, array::begin( Bytecode ), array::size( Bytecode ) ); - - CHECK( Result ); - CHECK_EQUAL( (unsigned)4, array::size( Stack ) ); - GET_ITEM( Stack, s32, PushResult ); - CHECK_EQUAL( 666, PushResult ); - } -} - -SUITE( InterpreterAddI ) -{ - TEST_FIXTURE( InterpreterCleanSetup, AddI00 ) - { - ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI ); - ADD_CONSTANT( Bytecode, s32, 0 ); - ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI ); - ADD_CONSTANT( Bytecode, s32, 0 ); - ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_AddI ); - - bool Result = interpreter::Run( Interp, array::begin( Bytecode ), array::size( Bytecode ) ); - - CHECK( Result ); - CHECK_EQUAL( (unsigned)4, array::size( Stack ) ); - GET_ITEM( Stack, s32, ResultValue ); - CHECK_EQUAL( 0, ResultValue ); - } - - TEST_FIXTURE( InterpreterCleanSetup, AddI42666 ) - { - ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI ); - ADD_CONSTANT( Bytecode, s32, 42 ); - ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI ); - ADD_CONSTANT( Bytecode, s32, 666 ); - ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_AddI ); - - bool Result = interpreter::Run( Interp, array::begin( Bytecode ), array::size( Bytecode ) ); - - CHECK( Result ); - CHECK_EQUAL( (unsigned)4, array::size( Stack ) ); - - GET_ITEM( Stack, s32, ResultValue ); - CHECK_EQUAL( 708, ResultValue ); - } -} +#include "interpretor_tests.h" +#include "token_generator_tests.h" int main( int, char const *[] ) { diff --git a/Dev/src/fry_script_test/token_generator_tests.h b/Dev/src/fry_script_test/token_generator_tests.h new file mode 100644 index 0000000..165df55 --- /dev/null +++ b/Dev/src/fry_script_test/token_generator_tests.h @@ -0,0 +1,96 @@ +#include "UnitTest++.h" +#include +#include + +using namespace fry_script; +using namespace memory; + +#include + +/// Helper that compares a char*[] with a array, so that we can run tests easily +#define CHECK_ARRAY_STRING_EQUAL( EXPECTED_RESULT, ACTUAL_RESULT ) \ +{ \ + const u32 EXPECTED_SIZE = sizeof( EXPECTED_RESULT ) / sizeof( EXPECTED_RESULT[0] ); \ + CHECK_EQUAL( EXPECTED_SIZE, array::size( ACTUAL_RESULT ) ); \ + if( EXPECTED_SIZE == array::size( ACTUAL_RESULT ) ) \ + { \ + TokenGenerator Generator; \ + const char* NextToken = NULL; \ + size_t Idx = 0; \ + while( token_generator::GetNextToken( &Generator, ACTUAL_RESULT, &NextToken ) ) \ + { \ + CHECK_EQUAL( 0, strcmp( NextToken, EXPECTED_RESULT[Idx++] ) ); \ + } \ + } \ +} + +struct TokenGeneratorCleanSetup +{ + TokenGeneratorCleanSetup() : + GeneratedTokens( memory_globals::default_allocator() ) + { + } + + ~TokenGeneratorCleanSetup() + { + array::clear( GeneratedTokens ); + } + + Array GeneratedTokens; +}; + +SUITE( TokenGenerator ) +{ + TEST_FIXTURE( TokenGeneratorCleanSetup, EmptyString ) + { + const char* Code = ""; + bool Result = token_generator::GenerateTokens( Code, GeneratedTokens ); + + CHECK( Result ); + CHECK_EQUAL( (u32)0, array::size( GeneratedTokens ) ); + } + + TEST_FIXTURE( TokenGeneratorCleanSetup, SingleToken ) + { + const char* Code = "SingleToken"; + const char* ExpectedResult[] = + { + "SingleToken" + }; + + bool Result = token_generator::GenerateTokens( Code, GeneratedTokens ); + + CHECK( Result ); + CHECK_ARRAY_STRING_EQUAL( ExpectedResult, GeneratedTokens ); + } + + TEST_FIXTURE( TokenGeneratorCleanSetup, TwoTokens ) + { + const char* Code = "FirstToken SecondToken"; + const char* ExpectedResult[] = + { + "FirstToken", + "SecondToken" + }; + + bool Result = token_generator::GenerateTokens( Code, GeneratedTokens ); + + CHECK( Result ); + CHECK_ARRAY_STRING_EQUAL( ExpectedResult, GeneratedTokens ); + } + + TEST_FIXTURE( TokenGeneratorCleanSetup, SpaceAtEnd ) + { + const char* Code = "FirstToken SecondToken "; + const char* ExpectedResult[] = + { + "FirstToken", + "SecondToken" + }; + + bool Result = token_generator::GenerateTokens( Code, GeneratedTokens ); + + CHECK( Result ); + CHECK_ARRAY_STRING_EQUAL( ExpectedResult, GeneratedTokens ); + } +} \ No newline at end of file