* Start on TokenGenerator

* Moved out testcases to own headers
MarkusRannare · Feb 17, 2013 · 97e743b · 97e743b
1 parent c66a38c
commit 97e743b
Show file tree

Hide file tree

Showing 10 changed files with 354 additions and 105 deletions.
diff --git a/Dev/src/fry_script/fry_script.vcxproj b/Dev/src/fry_script/fry_script.vcxproj
@@ -88,9 +88,12 @@
     <ClInclude Include="interpreter_types.h" />
     <ClInclude Include="opcodes.h" />
     <ClInclude Include="stack_manipulation.h" />
+    <ClInclude Include="token_generator.h" />
+    <ClInclude Include="token_generator_types.h" />
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="interpreter.cpp" />
+    <ClCompile Include="token_generator.cpp" />
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">

diff --git a/Dev/src/fry_script/fry_script.vcxproj.filters b/Dev/src/fry_script/fry_script.vcxproj.filters
@@ -33,10 +33,19 @@
     <ClInclude Include="stack_manipulation.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="token_generator.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="token_generator_types.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="interpreter.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="token_generator.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
 </Project>
diff --git a/Dev/src/fry_script/token_generator.cpp b/Dev/src/fry_script/token_generator.cpp
@@ -0,0 +1,74 @@
+#include "token_generator.h"
+#include <core/array.h>
+#include <core/memory.h>
+
+using namespace foundation;
+
+static const char CharsToIgnore[] =
+{
+	// Whitespaces
+	' ',
+	'\t',
+	'\n',
+	'\r'
+};
+
+#include <iostream>
+
+namespace fry_script
+{
+	namespace token_generator
+	{
+		bool GenerateTokens( const char* Code, Array<GeneratedToken>& out_Tokens )
+		{
+			GeneratedToken Tok;
+			size_t LastFoundIdx = 0;
+
+			Tok.Location = Code;
+			Tok.Length = 0;
+
+			size_t Idx = 0;
+			for( ; Idx < strlen( Code ); ++Idx )
+			{
+				if( Code[Idx] == ' ' )
+				{
+					Tok.Length = Idx - LastFoundIdx;
+
+					array::push_back( out_Tokens, Tok );
+
+					LastFoundIdx = Idx;
+					Tok.Location = &Code[Idx+1];
+					Tok.Length = 0;
+				}
+			}
+
+			Tok.Length = Idx - LastFoundIdx;
+			if( Tok.Length > 0 )
+			{
+				array::push_back( out_Tokens, Tok );
+			}
+
+			return true;
+		}
+
+		bool GetNextToken( TokenGenerator* Generator, Array<GeneratedToken>& Tokens, const char** out_NextToken )
+		{
+			size_t CurrentIndex = Generator->CurrentIndex++;
+			if( CurrentIndex >= array::size( Tokens ) )
+			{
+				Generator->CurrentIndex = -1;
+				memory::mem_zero( Generator->TempToken, sizeof( Generator->TempToken ) );
+
+				return false;
+			}
+
+			const GeneratedToken& CurrentToken = Tokens[CurrentIndex];
+
+			memcpy_s( Generator->TempToken, sizeof( Generator->TempToken ), CurrentToken.Location, CurrentToken.Length );
+			Generator->TempToken[CurrentToken.Length] = '\0';
+			*out_NextToken = Generator->TempToken;
+
+			return true;
+		}
+	}
+}
diff --git a/Dev/src/fry_script/token_generator.h b/Dev/src/fry_script/token_generator.h
@@ -0,0 +1,22 @@
+#ifndef FRY_SCRIPT_TOKEN_GENERATOR_H
+#define FRY_SCRIPT_TOKEN_GENERATOR_H
+
+#include <core/collection_types.h>
+#include "token_generator_types.h"
+
+namespace fry_script
+{
+	namespace token_generator
+	{
+		/// Generates tokens from code
+		/// \param Code - the code to generator tokens from
+		/// \param Allocator - the allocator that will allocate the tokens in out_Tokens
+		/// \param out_Tokens - the tokes generated from Code
+		bool GenerateTokens( const char* Code, foundation::Array<GeneratedToken>& out_Tokens );
+
+		/// Copies a single token
+		bool GetNextToken( TokenGenerator* Generator, foundation::Array<GeneratedToken>& Tokens, const char** out_NextToken );
+	}
+}
+
+#endif
diff --git a/Dev/src/fry_script/token_generator_types.h b/Dev/src/fry_script/token_generator_types.h
@@ -0,0 +1,31 @@
+#ifndef FRY_SCRIPT_TOKEN_GENERATOR_TYPES
+#define FRY_SCRIPT_TOKEN_GENERATOR_TYPES
+
+namespace fry_script
+{
+	/// Largest size of a allowed token
+	const s32 MaxTokenSize = 255;
+
+	/// Struct that describes the tokens generated from GenerateTokens
+	/// It contains a pointer into the char* Code that is sent into the
+	/// GenerateToken function
+	struct GeneratedToken
+	{
+		const char* Location;
+		u32	Length;
+	};
+
+	/// The state of a TokenGenerator (for multithread support)
+	struct TokenGenerator
+	{
+		TokenGenerator() :
+			CurrentIndex( 0 )
+		{
+		}
+
+		int CurrentIndex;
+		char TempToken[MaxTokenSize];
+	};
+}
+
+#endif
diff --git a/Dev/src/fry_script_test/fry_script_test.vcxproj b/Dev/src/fry_script_test/fry_script_test.vcxproj
@@ -13,6 +13,10 @@
   <ItemGroup>
     <ClCompile Include="main.cpp" />
   </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="interpretor_tests.h" />
+    <ClInclude Include="token_generator_tests.h" />
+  </ItemGroup>
   <PropertyGroup Label="Globals">
     <ProjectGuid>{97E8EAAA-7115-406C-9F98-DD88CAB931F2}</ProjectGuid>
     <Keyword>Win32Proj</Keyword>

diff --git a/Dev/src/fry_script_test/fry_script_test.vcxproj.filters b/Dev/src/fry_script_test/fry_script_test.vcxproj.filters
@@ -19,4 +19,12 @@
       <Filter>Source Files</Filter>
     </ClCompile>
   </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="interpretor_tests.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="token_generator_tests.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+  </ItemGroup>
 </Project>
diff --git a/Dev/src/fry_script_test/interpretor_tests.h b/Dev/src/fry_script_test/interpretor_tests.h
@@ -0,0 +1,105 @@
+#include "UnitTest++.h"
+
+#include <core/stack.h>
+#include <fry_script/interpreter.h>
+#include <fry_script/stack_manipulation.h>
+
+using namespace foundation;
+using namespace fry_core;
+using namespace fry_script;
+
+struct InterpreterCleanSetup
+{
+	InterpreterCleanSetup() :
+	Interp( memory_globals::default_allocator() ),
+		Bytecode( memory_globals::default_allocator() ),
+		Stack( Interp._Stack )
+	{
+	}
+
+	~InterpreterCleanSetup()
+	{
+	}
+
+	#pragma warning( disable: 4100 )
+	InterpreterCleanSetup& operator=( const InterpreterCleanSetup& Other )
+	{
+		// Should never be here, but can't use CHECK as FRY_CORE::CHECK conflicts with C++TEST::CHECK
+		return *this;
+	}
+	#pragma warning( default: 4100 )
+
+	Interpreter Interp;
+	Array<u8> Bytecode;
+	// Helper to prevent us from typing Interp._Stack all the time
+	Array<u8>& Stack;
+};
+
+SUITE( InterpreterPushI )
+{
+	TEST_FIXTURE( InterpreterCleanSetup, PushI0 )
+	{
+		CHECK_EQUAL( (unsigned)0, array::size( Stack ) );
+
+		ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI );
+		ADD_CONSTANT( Bytecode, s32, 0 );
+
+		bool Result = interpreter::Run( Interp, array::begin( Bytecode ), array::size( Bytecode ) );
+
+		CHECK( Result );
+		CHECK_EQUAL( (unsigned)4, array::size( Stack ) );
+		GET_ITEM( Stack, s32, PushResult );
+		CHECK_EQUAL( 0, PushResult );
+	}
+
+	TEST_FIXTURE( InterpreterCleanSetup, PushI666 )
+	{
+		CHECK_EQUAL( (unsigned)0, array::size( Stack ) );
+
+		ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI );
+		ADD_CONSTANT( Bytecode, s32, 666 );
+
+		bool Result = interpreter::Run( Interp, array::begin( Bytecode ), array::size( Bytecode ) );
+
+		CHECK( Result );
+		CHECK_EQUAL( (unsigned)4, array::size( Stack ) );
+		GET_ITEM( Stack, s32, PushResult );
+		CHECK_EQUAL( 666, PushResult );
+	}
+}
+
+SUITE( InterpreterAddI )
+{
+	TEST_FIXTURE( InterpreterCleanSetup, AddI00 )
+	{
+		ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI );
+		ADD_CONSTANT( Bytecode, s32, 0 );
+		ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI );
+		ADD_CONSTANT( Bytecode, s32, 0 );
+		ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_AddI );
+
+		bool Result = interpreter::Run( Interp, array::begin( Bytecode ), array::size( Bytecode ) );
+
+		CHECK( Result );
+		CHECK_EQUAL( (unsigned)4, array::size( Stack ) );
+		GET_ITEM( Stack, s32, ResultValue );
+		CHECK_EQUAL( 0, ResultValue );
+	}
+
+	TEST_FIXTURE( InterpreterCleanSetup, AddI42666 )
+	{
+		ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI );
+		ADD_CONSTANT( Bytecode, s32, 42 );
+		ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_PushI );
+		ADD_CONSTANT( Bytecode, s32, 666 );
+		ADD_CONSTANT( Bytecode, u8, (u8)fry_script::OC_AddI );
+
+		bool Result = interpreter::Run( Interp, array::begin( Bytecode ), array::size( Bytecode ) );
+
+		CHECK( Result );
+		CHECK_EQUAL( (unsigned)4, array::size( Stack ) );
+
+		GET_ITEM( Stack, s32, ResultValue );
+		CHECK_EQUAL( 708, ResultValue );
+	}
+}