added retrospective learning, monkeyed with tokeniser, started implem…

…enting greet new

brains.py

@@ -10,40 +10,12 @@
 from google.appengine.api import namespace_manager
 import settings
 import state
+import types
 
 
 # don't take any longer than this to process.
 TIME_LIMIT = datetime.timedelta( minutes=9 )
 
-def digest_user( api, deadline, mm_twitteruser ):
-
-	last_id = mm_twitteruser.last_id
-	num_statuses = 20
-	if last_id is None:
-		# if we haven't seen this user before, get more statuses for better input
-		num_statuses = 100
-
-	if mm_twitteruser.id_str:
-		statuses = api.user_timeline( count=num_statuses, user_id=mm_twitteruser.id_str, since_id=last_id, include_rts=False )	
-	elif mm_twitteruser.screen_name:
-		statuses = api.user_timeline( count=num_statuses, screen_name=mm_twitteruser.screen_name, since_id=last_id, include_rts=False )	
-
-	statuses_digested = 0
-	if statuses is not None:
-		if len(statuses) > 0:
-			worker = verbivorejr.VerbivoreWorker()
-			for status in reversed(statuses): # reversed so we start at the oldest, in case we have to abort 
-				last_id = status.id_str
-				worker.digest( status.text, deadline )
-				statuses_digested = statuses_digested + 1
-				if datetime.datetime.now() >= deadline:
-					logging.debug( "brains.digest_user(), hit deadline at %d statuses" % statuses_digested )
-					break
-			worker.put( deadline )
-			mm_twitteruser.last_id = last_id
-			mm_twitteruser.put()
-
-	return statuses_digested
 
 def post_tweet( api, tweet, in_reply_to_status_id=None ):
 	if tweet is not None:
@@ -54,13 +26,6 @@ def post_tweet( api, tweet, in_reply_to_status_id=None ):
 		except Exception, err:
 			logging.debug( "brains.run(): error from twitter api: %s" % err )
 
-def learn_from_ids( api, guru_ids, deadline ):
-	statuses_digested = 0
-	for guru_id in guru_ids:
-			guru = twitter.get_user( id_str=guru_id )
-			statuses_digested += digest_user( api, deadline, guru )	
-	return( statuses_digested )
-
 def run( creds, force_tweet=False, debug=False ):	
 
 	if not debug:
@@ -89,25 +54,23 @@ def run( creds, force_tweet=False, debug=False ):
 	namespace_manager.set_namespace( creds.screen_name )
 
 	logging.debug( "brains.run(): learning_style is: %s" % learning_style )
+	worker = verbivorejr.VerbivoreWorker( api, bot_settings )
+	worker.deadline = deadline
 	if learning_style == constants.learning_style_oneuser:
 		# learn from one user
 		guru_name = bot_settings.learning_guru
 		guru = twitter.get_user( screen_name=guru_name )
-		statuses_digested = digest_user( api, deadline, guru )
+		statuses_digested = worker.digest_user( guru )
 	elif learning_style == constants.learning_style_following:
 		guru_ids = api.friends_ids( stringify_ids=True )
-		statuses_digested = learn_from_ids( api, guru_ids, deadline )
+		statuses_digested = worker.digest_ids( guru_ids )
 	elif learning_style == constants.learning_style_followers:
 		guru_ids = api.followers_ids( stringify_ids=True )
-		statuses_digested = learn_from_ids( api, guru_ids, deadline )
+		statuses_digested = worker.digest_ids( guru_ids )
 
-	logging.debug( "brains.run(): digested %d new statuses" % statuses_digested )
+	worker.put()
 
-	# check deadline
-	if datetime.datetime.now() >= deadline:
-		logging.debug( "brains.run(): aborted after put()'ing worker, deadline is looming." )
-		taskqueue.add( url="/%s/run" % api.me().screen_name )
-		return
+	logging.debug( "brains.run(): digested %d new statuses" % statuses_digested )
 
 	# only continue if chance is met
 	if bot_settings.tweet_chance < random.random() and force_tweet is False:
@@ -125,14 +88,21 @@ def run( creds, force_tweet=False, debug=False ):
 	elif bot_settings.locquacity_speakonnew and statuses_digested > 0 :
 		logging.debug( "brains.run(): locquacity_speakonnew, statuses_digested: %s" % statuses_digested )
 		do_tweet = True
+
+	# check deadline, defer tweeting if necessary
+	if datetime.datetime.now() >= deadline:
+		logging.debug( "brains.run(): aborted after put()'ing worker, deadline is looming." )
+		taskqueue.add( url="/%s/run" % api.me().screen_name )
+		return
 
 	queen = verbivorejr.VerbivoreQueen()
+	queen.deadline = deadline
 
 	if do_tweet:
 		tweet = None
-		safety = 3
+		safety = 10
 		while tweet is None and safety > 0:
-			tweet = queen.secrete( 130, deadline )
+			tweet = queen.secrete( 130 )
 			safety = safety - 1
 		if tweet is not None:
 			tweet = verbivorejr.uc_first( tweet )
@@ -142,7 +112,7 @@ def run( creds, force_tweet=False, debug=False ):
 
 	if bot_settings.locquacity_reply:
 
-		last_replied_id = creds.last_replied_id	
+		last_replied_id = bot_state.last_replied_id	
 		logging.debug( "brains.run(): last_replied_id is %s" % last_replied_id )
 		mentions = api.mentions( since_id=last_replied_id )
 		logging.debug( "-> %d mentions" % len(mentions) )
@@ -151,6 +121,9 @@ def run( creds, force_tweet=False, debug=False ):
 		last_timestamp = None
 		for mention in mentions:
 
+			if datetime.datetime.now() >= deadline:
+				break
+
 			# only reply when we've been directly addressed
 			#if mention.text[:len(my_name)] != my_name:
 			#	break
@@ -162,7 +135,7 @@ def run( creds, force_tweet=False, debug=False ):
 				logging.debug( "--> generate reply, safety=%d" % safety )
 				if datetime.datetime.now() >= deadline:
 					break
-				tweet = queen.secrete_reply( mention.text, 130 - len(reply), deadline )
+				tweet = queen.secrete_reply( mention.text, 130 - len(reply) )
 				safety = safety -1
 
 			if tweet is not None:
@@ -177,7 +150,33 @@ def run( creds, force_tweet=False, debug=False ):
 				last_replied_id = mention.id_str
 				last_timestamp = this_timestamp
 
-		creds.last_replied_id = last_replied_id
+		bot_state.last_replied_id = last_replied_id
+		bot_state.put()
+
+
+	if bot_settings.locquacity_greetnew:
+
+		if datetime.datetime.now() >= deadline:
+			logging.debug( "brains.run(): aborted before greeting new followers, deadline is looming." )
+			return
+
+		new_follower_ids = None
+		stored_follower_ids = creds.follower_ids
+		api_follower_ids = api.followers_ids()
+		if stored_follower_ids is None:
+			new_follower_ids = api_follower_ids
+		else:
+			new_follower_ids = []
+			for api_follower_id in api_follower_ids:
+				if api_follower_id not in stored_follower_ids:
+					new_follower_ids.append( api_follower_id )
+
+		if new_follower_ids is not None and len(new_follower_ids) > 0:
+			logging.debug( "brains.run(): new_follower_ids: %s" % new_follower_ids )
+		else:
+			logging.debug( "brains.run(): no new followers" )
+
+		creds.follower_ids = api_follower_ids
 		creds.put()
 
 	now = datetime.datetime.now()

index.yaml

@@ -18,12 +18,12 @@ indexes:
 
 - kind: VBWordForwardLink
   properties:
-  - name: root_word
+  - name: next_word
   - name: frequency
     direction: desc
 
 - kind: VBWordForwardLink
   properties:
-  - name: next_word
+  - name: root_word
   - name: frequency
-    direction: desc
+    direction: desc

main.py

@@ -53,6 +53,7 @@ def get(self):
 class RunHandler( webapp.RequestHandler ):
 
 	def run_one( self, bot_name ):
+		logging.debug( "RunHandler.run_one: %s", bot_name )
 		creds = twitter.get_twitter_creds( bot_name )
 		force_tweet = self.request.get( "force_tweet") == "true"
 		debug = self.request.get( "debug" ) == "true"
@@ -98,6 +99,13 @@ def path_for_template( template_name ):
 
 class SettingsHandler( webapp.RequestHandler ):
 
+	def render_notloggedin( self ):
+		template_values = {}
+		template_values[ 'message' ] = "Sorry, you need to be logged in to a Google account to create bots."
+		template_values[ 'login_url' ] = users.create_login_url( self.request.path )
+		template_path = path_for_template( "notloggedin.html" )
+		self.response.out.write( template.render( template_path, template_values ) )	
+
 	def render_template( self, creds, bot_settings=None, values=None ):
 
 		template_values = {}
@@ -138,12 +146,16 @@ def render_template( self, creds, bot_settings=None, values=None ):
 			elif bot_settings.learning_style == constants.learning_style_list:
 				template_values[ 'learnfrom_list_checked' ] = "checked"
 
+			if bot_settings.learn_retrospectively: 
+				template_values[ 'learn_retrospectively_checked' ] = "checked"
 			if bot_settings.locquacity_onschedule: 
 				template_values[ 'locquacity_onschedule_checked' ] = "checked"
 			if bot_settings.locquacity_reply:
 				template_values[ 'locquacity_reply_checked' ] = "checked"
 			if bot_settings.locquacity_speakonnew:
 				template_values[ 'locquacity_speakonnew_checked' ] = "checked"
+			if bot_settings.locquacity_greetnew:
+				template_values[ 'locquacity_greetnew_checked' ] = "checked"
 
 			template_path = path_for_template( "settings.html" )
 
@@ -159,24 +171,35 @@ def get( self, bot_name ):
 		if self.authenticate_user( creds ):
 			self.render_template( creds ) 
 		else:
-			path = path_for_template( "notowner.html" )
-			self.response.out.write( template.render( path, {} ) )
+			self.render_notloggedin()
 
 	# form data has been posted, process it
 	def post( self, bot_name ):
 
 		creds = twitter.get_twitter_creds( bot_name )
 
 		if not self.authenticate_user( creds ):
-			path = path_for_template( "notowner.html" )
-			self.response.out.write( template.render( path, template_values ) )
+			self.render_notloggedin()
 		else:
 			bot_settings = settings.get_settings( creds )
 			bot_settings.learning_style = self.request.get( 'learnfrom' )
 			bot_settings.learning_guru = self.request.get( 'guru_name' )
 			bot_settings.locquacity_onschedule = self.request.get( 'locquacity_onschedule' ) == "true"
 			bot_settings.locquacity_reply = self.request.get( 'locquacity_reply' ) == "true"
 			bot_settings.locquacity_speakonnew = self.request.get( 'locquacity_speakonnew' ) == "true"
+			bot_settings.learn_retrospectively = self.request.get( 'learn_retrospectively' ) == "true"
+
+			gn = self.request.get( 'locquacity_greetnew' ) == "true"
+			logging.debug( 'SettingsHandler.post(): locquacity_greetnew=%s, bot_settings.locquacity_greetnew=%s' % (gn, bot_settings.locquacity_greetnew) )
+			if gn and not bot_settings.locquacity_greetnew:
+				logging.debug( '-> fetch follower ids' )
+				api = twitter.get_api( creds )
+				follower_ids = api.followers_ids()
+				logging.debug( '--> follower_ids=%s' % "|".join( follower_ids ) )
+				creds.follower_ids = follower_ids
+				creds.put()
+			bot_settings.locquacity_greetnew = gn
+
 			tweet_frequency = self.request.get( 'tweet_frequency' )
 			if tweet_frequency is not None and len(tweet_frequency) > 0:
 				bot_settings.tweet_frequency = float( tweet_frequency )
@@ -196,9 +219,8 @@ def get(self):
 		user = users.get_current_user()
 
 		if user is None:
-			msg = "Sorry, you need to be logged in to a Google account to create bots.<br/>"
-			msg += "<a href=\"%s\">Sign in or register</a>." % users.create_login_url("/new")
-			template_values[ 'message' ] = msg
+			template_values[ 'login_url' ] = users.create_login_url("/new")
+			self.response.out.write( template.render( "notloggedin.html", template_values ) )
 		else:
 			redirect_url = None
 			tw_error = None
@@ -215,8 +237,7 @@ def get(self):
 			else:
 				template_values[ "twitter_auth" ] = redirect_url
 
-		path = path_for_template( "new.html" )
-		self.response.out.write( template.render( path, template_values ) )
+			self.response.out.write( template.render( "new.html", template_values ) )
 
 class OAuthReturnHandler( webapp.RequestHandler ):
 	def get(self):

settings.py

@@ -10,6 +10,9 @@ class MMSettings( db.Model ):
 	locquacity_speakonnew = db.BooleanProperty( required=True, default=False )
 	tweet_frequency = db.FloatProperty( required=True, default=1.0 )
 	tweet_chance = db.FloatProperty( required=True, default=1.0 )
+	greet_new_followers = db.BooleanProperty( default=False )
+	learn_retrospectively = db.BooleanProperty( default=False )
+	locquacity_greetnew = db.BooleanProperty( default=False )
 
 def get_settings( creds ):
 	settings = MMSettings.all()

state.py

@@ -3,8 +3,9 @@
 
 class MMState( db.Model ):
 	creds = db.ReferenceProperty( twitter.MMTwitterCreds )
+	last_replied_id = db.StringProperty()
 	last_run = db.DateTimeProperty()
-
+	
 def get_state( creds ):
 	q = MMState.all()
 	q.filter( "creds = ", creds )

templates/notloggedin.html

@@ -0,0 +1,8 @@
+<html>
+	<head>
+	</head>
+	<body>
+		{{message}}<br/>
+		<a href="{{login_url}}">Sign in or register</a>
+	</body>
+</html>

templates/settings.html

@@ -60,6 +60,8 @@ <h1>Saved</h1>
 			</fieldset>
 			<input type="radio" name="learnfrom" value="followers" onclick="onLearnRadioClicked(this);" {{learnfrom_followers_checked}}/>
 			Learn from my followers
+			<input type="checkbox" name="learn_retrospectively" value="true" {{learn_retrospectively_checked}}/>
+			Learn retrospectively
 		</fieldset>
 		<fieldset id="fieldset_locquacity">
 			<legend>Locquacity</legend>
@@ -73,6 +75,8 @@ <h1>Saved</h1>
 			</fieldset>
 			<input type="checkbox" name="locquacity_reply" value="true" {{locquacity_reply_checked}}/>
 			Reply to @mentions and direct messages
+			<input type="checkbox" name="locquacity_greetnew" value="true" {{locquacity_greetnew_checked}}/>
+			Greet new followers
 		</fieldset>
 		<input type="submit" value="Save" />
 	</form>