Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cache powerups #117

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 40 additions & 5 deletions axiom/item.py
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,12 @@ class Empowered(object):
are being loaded and with a list of powerups found in C{store}. The
return value is the powerup. These are used only by the callable
interface adaption API, not C{powerupsFor}.

@type _dbPowerups: C{dict}
@ivar _dbPowerups: Mapping from interface classes to lists of (priority,
item). This is only populated on the first call to L{powerupsFor} for a
particular interface; thereafter, it will be updated when powerups are
added and removed, to avoid loading the powerups from the database again.
"""

aggregateInterfaces = {
Expand Down Expand Up @@ -270,6 +276,11 @@ def powerUp(self, powerup, interface=None, priority=0):
powerup=powerup)
forc.priority = priority

pups = self._dbPowerups.get(interface)
if pups is not None:
pups.append((priority, powerup))
pups.sort(key=lambda (prio, p): prio, reverse=True)


def powerDown(self, powerup, interface=None):
"""
Expand Down Expand Up @@ -298,6 +309,10 @@ def powerDown(self, powerup, interface=None):
AND(_PowerupConnector.item == self,
_PowerupConnector.interface == unicode(qual(interface)),
_PowerupConnector.powerup == powerup)):
pups = self._dbPowerups.get(interface)
if pups is not None:
for elem in [elem for elem in pups if elem[1] is cable.powerup]:
pups.remove(elem)
cable.deleteFromStore()
return
raise ValueError("Not powered up for %r with %r" % (interface,
Expand Down Expand Up @@ -341,6 +356,27 @@ def powerupsFor(self, interface):
inMemoryPowerup = self._inMemoryPowerups.get(interface, None)
if inMemoryPowerup is not None:
yield inMemoryPowerup

try:
pups = self._dbPowerups[interface]
except KeyError:
pups = list(self._powerupsFor(interface))
else:
pups = [(prio, p) for (prio, p) in pups if p._currentlyValidAsReferentFor(self.store)]
self._dbPowerups[interface] = pups

for (prio, pup) in pups:
indirector = IPowerupIndirector(pup, None)
if indirector is not None:
yield indirector.indirect(interface)
else:
yield pup


def _powerupsFor(self, interface):
"""
Retrieve powerups from the database.
"""
name = unicode(qual(interface), 'ascii')
for cable in self.store.query(
_PowerupConnector,
Expand All @@ -352,11 +388,7 @@ def powerupsFor(self, interface):
# this powerup was probably deleted during an upgrader.
cable.deleteFromStore()
else:
indirector = IPowerupIndirector(pup, None)
if indirector is not None:
yield indirector.indirect(interface)
else:
yield pup
yield (cable.priority, pup)

def interfacesFor(self, powerup):
"""
Expand Down Expand Up @@ -495,6 +527,8 @@ class Item(Empowered, slotmachine._Strict):

# A mapping from interfaces to in-memory powerups.
_inMemoryPowerups = inmemory()
# A mapping from interfaces to in-database powerups.
_dbPowerups = inmemory()

def _currentlyValidAsReferentFor(self, store):
"""
Expand Down Expand Up @@ -587,6 +621,7 @@ def __subinit__(self, **kw):
"""
self._axiom_service = None
self._inMemoryPowerups = {}
self._dbPowerups = {}
self.__dirty__ = {}
to__store = kw.pop('__store', None)
to__everInserted = kw.pop('__everInserted', False)
Expand Down
9 changes: 8 additions & 1 deletion axiom/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -652,7 +652,13 @@ def itemsToDelete(attr):
self.tableClass, itemsToDelete):
it.deleteFromStore()

# actually run the DELETE for the items in this query.
# Slow-delete items in this query that are in the object cache.
for storeID in self.getColumn('storeID'):
if self.store.objectCache.has(storeID):
obj = self.store.objectCache.get(storeID)
obj.deleteFromStore()

# Run the DELETE for the rest of the items in this query.
self._runQuery('DELETE', "")

class MultipleItemQuery(BaseQuery):
Expand Down Expand Up @@ -1113,6 +1119,7 @@ def __init__(self, dbdir=None, filesdir=None, debug=False, parent=None, idInPare
self.execTimes = []

self._inMemoryPowerups = {}
self._dbPowerups = {}

self._attachedChildren = {} # database name => child store object

Expand Down
104 changes: 104 additions & 0 deletions axiom/test/test_powerup.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import gc

from twisted.trial import unittest

Expand Down Expand Up @@ -104,6 +105,20 @@ def __getPowerupInterfaces__(self, pifs):
return 'not a list of pairs'



class CountingItem(Item):
"""
An Item that increments a counter every time it is loaded from the
database.
"""
stuff = integer()

count = 0

def activate(self):
CountingItem.count += 1


class PowerUpTest(unittest.TestCase):

def testBasicPowerups(self):
Expand Down Expand Up @@ -238,6 +253,95 @@ def testNoIndirectedIndirection(self):
self.assertEqual(list(s.powerupsFor(IPowerupIndirector)), [])


def test_powerDownDeleted(self):
"""
If a powerup is deleted without powering down first, it should be
powered down automatically.
"""
s = Store()
mm = Summer(store=s)
s.powerUp(mm, ISumProducer)
mm.deleteFromStore()
self.assertEqual(list(s.powerupsFor(ISumProducer)), [])



class CachingTests(unittest.TestCase):
"""
Tests for powerup caching behaviour.
"""
def test_powerupCaching(self):
"""
Powerups for an item should only be loaded from database once so long
as that item remains in memory.
"""
s = Store()
CountingItem.count = 0

ci = CountingItem(store=s)
s.powerUp(ci, ISumProducer)
self.assertEqual(CountingItem.count, 1)
del ci
gc.collect()

pups = s.powerupsFor(ISumProducer)
self.assertEqual(CountingItem.count, 1)


def test_deleteCachedPowerup(self):
"""
A cached powerup that is deleted should be removed from the list of
installed powerups.
"""
s = Store()
ci = CountingItem(store=s)
s.powerUp(ci, ISumProducer)

pups = list(s.powerupsFor(ISumProducer))
self.assertEqual(pups, [ci])

ci.deleteFromStore()
pups2 = list(s.powerupsFor(ISumProducer))
self.assertEqual(pups2, [])


def test_newCachedPowerup(self):
"""
Adding a new powerup when there are already cached powerups should add
the powerup to the cached list in the correct order.
"""
s = Store()
ci = CountingItem(store=s)
s.powerUp(ci, ISumProducer)

pups = list(s.powerupsFor(ISumProducer))
self.assertEqual(pups, [ci])

ci2 = CountingItem(store=s)
s.powerUp(ci2, ISumProducer, priority=50)
pups2 = list(s.powerupsFor(ISumProducer))
self.assertEqual(pups2, [ci2, ci])


def test_newCachedPowerupIndirection(self):
"""
Adding a new indirect powerup when there are already cached powerups
should add the powerup to the cached list.
"""
s = Store()
s.powerUp(
SubtractThree(
store=s, valueHaver=SumContributor(store=s, value=5)),
IValueHaver)
self.assertEqual(len(list(s.powerupsFor(IValueHaver))), 1)

s.powerUp(
SubtractThree(
store=s, valueHaver=SumContributor(store=s, value=5)),
IValueHaver)
self.assertEqual(len(list(s.powerupsFor(IValueHaver))), 2)



from twisted.application.service import IService, Service

Expand Down
16 changes: 16 additions & 0 deletions axiom/test/test_reference.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,22 @@ def testBadReferenceNone(self):
(referent,) = list(store.query(SimpleReferent))
self.assertEqual(referent.ref, None)


def test_badReferencedNoneCache(self):
"""
Test that accessing a broken reference to an Item that has been
deleted, but is still in the object cache, correctly nullifies the
attribute.
"""
store = Store()
def _test():
referee = Referee(store=store, topSecret=0)
referent = SimpleReferent(store=store, ref=referee)
referee.deleteFromStore()
self.assertEqual(referent.ref, None)
store.transact(_test)


def testBadReferenceNoneLoading(self):
"""
Test that accessing a broken reference on an Item that has not yet been
Expand Down
16 changes: 16 additions & 0 deletions axiom/test/test_xatop.py
Original file line number Diff line number Diff line change
Expand Up @@ -652,6 +652,11 @@ class StricterItem(item.Item):
aRef = attributes.reference(allowNone=False)



class ReferringItem(item.Item):
aRef = attributes.reference(whenDeleted=attributes.reference.NULLIFY)


class AttributeTests(unittest.TestCase):
def testGetAttribute(self):
s = store.Store()
Expand Down Expand Up @@ -957,6 +962,17 @@ def test_slowBatchDeleteBecauseDeletedFromStore(self):
self.assertEqual(DeleteFromStoreTrackingItem.deletedTimes, 1)


def test_batchDeleteReference(self):
"""
Ensure that a reference to an Item deleted in a batch delete, but still
in the object cache, is nullified correctly.
"""
referee = AttributefulItem(store=self.store, withoutDefault=42)
referent = ReferringItem(store=self.store, aRef=referee)
self.store.query(AttributefulItem).deleteFromStore()
self.assertEqual(referent.aRef, None)


# Item types we will use to change the underlying database schema (by creating
# them).
class ConcurrentItemA(item.Item):
Expand Down
44 changes: 44 additions & 0 deletions benchmark/batch-deletion
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
#!/usr/bin/python

# Benchmark of batch Item deletion. Accepts no parameters. Reports one
# statistic, the number of milliseconds it takes to batch delete an Item.

import sys, time

from axiom.store import Store
from axiom.attributes import integer

import benchlib


def benchmark():
SomeItem = benchlib.itemTypeWithSomeAttributes([integer])

store = Store()
outerCounter = range(10)
innerCounter = range(1000)

def _create():
benchlib.createSomeItems(store, SomeItem, {}, innerCounter)

def _delete():
store.query(SomeItem).deleteFromStore()

timer = benchlib.StopWatch()
for i in outerCounter:
store.transact(_create)
timer.start()
store.transact(_delete)
timer.stop()

return (timer.duration) * 1000 / (len(outerCounter) * len(innerCounter))


def main(argv):
if len(argv) != 1:
raise SystemExit("Usage: %s" % (argv[0],))
print benchmark()


if __name__ == '__main__':
main(sys.argv)
18 changes: 18 additions & 0 deletions benchmark/benchlib.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,28 @@
Helper functions useful to more than one benchmark script.
"""

import time
from itertools import count

from axiom.item import Item
from axiom.attributes import integer

class StopWatch(object):
"""
A very basic stopwatch timer.
"""
def __init__(self):
self.duration = 0.0

def start(self):
self.started = time.time()

def stop(self):
self.finished = time.time()
self.duration += (self.finished - self.started)



typeNameCounter = count(0).next

def itemTypeWithSomeAttributes(attributeTypes):
Expand All @@ -22,6 +39,7 @@ class SomeItem(Item):
return SomeItem



def createSomeItems(store, itemType, values, counter):
"""
Create some instances of a particular type in a store.
Expand Down
Loading