From a7a64fc85984698a4cab6cf549f9ed1eff12ba81 Mon Sep 17 00:00:00 2001 From: Corey Oordt Date: Sun, 24 Mar 2013 11:51:59 -0500 Subject: [PATCH 1/9] Basic PEP 8 cleaning --- supertagging/__init__.py | 14 +- supertagging/admin.py | 78 +++--- supertagging/calais.py | 132 +++++---- supertagging/fields.py | 3 +- supertagging/handlers.py | 23 +- .../st_migrate_pickled_object_fields.py | 33 ++- .../management/commands/st_process_queue.py | 10 +- supertagging/managers.py | 7 +- supertagging/markup.py | 93 ++++--- supertagging/models.py | 180 ++++++------ supertagging/modules/__init__.py | 182 +++++++------ supertagging/settings.py | 114 ++++---- supertagging/tests.py | 10 +- supertagging/utils.py | 256 +++++------------- 14 files changed, 538 insertions(+), 597 deletions(-) diff --git a/supertagging/__init__.py b/supertagging/__init__.py index 39abad8..6b4a12f 100644 --- a/supertagging/__init__.py +++ b/supertagging/__init__.py @@ -1,11 +1,12 @@ __version_info__ = { 'major': 0, - 'minor': 5, - 'micro': 4, - 'releaselevel': 'final', + 'minor': 6, + 'micro': 0, + 'releaselevel': 'beta', 'serial': 1 } + def get_version(): vers = ["%(major)i.%(minor)i" % __version_info__, ] @@ -17,6 +18,7 @@ def get_version(): __version__ = get_version() + class AlreadyRegistered(Exception): """ An attempt was made to register a model more than once. @@ -60,9 +62,9 @@ def register(model, tag_descriptor_attr='supertags', # Add custom manager ModelTaggedItemManager().contribute_to_class(model, tagged_item_manager_attr) - + # Add a reverse generic relationship to supertaggeditems GenericRelation(SuperTaggedItem).contribute_to_class(model, 'supertaggeditem_set') - + # Finally register in registry - registry.append(model) \ No newline at end of file + registry.append(model) diff --git a/supertagging/admin.py b/supertagging/admin.py index 4e15f44..104a1b2 100644 --- a/supertagging/admin.py +++ b/supertagging/admin.py @@ -1,17 +1,16 @@ -import django from django.contrib import admin from django.contrib.contenttypes.models import ContentType -from django.utils.translation import ugettext, ugettext_lazy -from django.utils.encoding import force_unicode, smart_str +from django.utils.translation import ugettext +from django.utils.encoding import force_unicode from django.http import HttpResponseRedirect from supertagging.models import SuperTag, SuperTaggedItem, SuperTagRelation from supertagging.models import SuperTaggedRelationItem, SuperTagProcessQueue from supertagging.settings import INCLUDE_DISPLAY_FIELDS -from django.contrib.admin.views.main import (ChangeList, ALL_VAR, ORDER_VAR, - ORDER_TYPE_VAR, PAGE_VAR, SEARCH_VAR, TO_FIELD_VAR, - IS_POPUP_VAR, ERROR_FLAG) +from django.contrib.admin.views.main import (ChangeList, ALL_VAR, PAGE_VAR, + SEARCH_VAR, TO_FIELD_VAR, IS_POPUP_VAR, ERROR_FLAG) + class SupertagChangeList(ChangeList): """ @@ -30,7 +29,7 @@ def __init__(self, request, model, list_display, list_display_links, list_filter self.list_select_related = list_select_related self.list_per_page = list_per_page self.model_admin = model_admin - + # Get search parameters from the query string. try: self.page_num = int(request.GET.get(PAGE_VAR, 0)) @@ -44,7 +43,7 @@ def __init__(self, request, model, list_display, list_display_links, list_filter del self.params[PAGE_VAR] if ERROR_FLAG in self.params: del self.params[ERROR_FLAG] - + self.list_editable = list_editable self.order_field, self.order_type = self.get_ordering() self.query = request.GET.get(SEARCH_VAR, '') @@ -54,79 +53,82 @@ def __init__(self, request, model, list_display, list_display_links, list_filter self.filter_specs, self.has_filters = self.get_filters(request) self.pk_attname = self.lookup_opts.pk.attname + def lock_items(modeladmin, request, queryset): queryset.update(locked=True) lock_items.short_description = "Lock selected Queue Items" + def unlock_items(modeladmin, request, queryset): queryset.update(locked=False) unlock_items.short_description = "Unlock selected Queue Items" + class SuperTagAdmin(admin.ModelAdmin): list_display = ('name', 'enabled', 'substitute', 'stype') ordering = ('name', ) search_fields = ('stype', 'name', ) list_filter = ('stype', ) - + actions = ['disable_tag', 'enable_tag'] - - raw_id_fields = ['substitute',] + + raw_id_fields = ['substitute', ] if INCLUDE_DISPLAY_FIELDS: raw_id_fields.append('related') - + def disable_tag(self, request, queryset): message_bit = "" for tag in queryset: message_bit = "%s %s," % (message_bit, tag.name) - tag.enabled=False + tag.enabled = False tag.save() - + self.message_user(request, "Tag(s): %s were Disabled." % message_bit) disable_tag.short_description = "Disable selected tags" - - + def enable_tag(self, request, queryset): message_bit = "" for tag in queryset: message_bit = "%s %s," % (message_bit, tag.name) - tag.enabled=True + tag.enabled = True tag.save() - + self.message_user(request, "Tag(s): %s were Enabled." % message_bit) enable_tag.short_description = "Enable selected tags" - + + class SuperTaggedItemAdmin(admin.ModelAdmin): list_display = ('tag_name', 'tag_type', 'field', 'relevance_bar', 'ignore') # if django.VERSION[1] > 1: # list_filter = ('field', 'tag__stype') # else: # list_filter = ('field', ) - + #search_fields = ('tag__name',) raw_id_fields = ('tag',) list_editable = ('ignore',) - + class Media: css = {'all': ('css/supertagloading.css',)} js = ('js/jquery.loading.1.6.4.min.js',) - + def tag_name(self, obj): if INCLUDE_DISPLAY_FIELDS: return obj.tag.display_name return obj.tag.name - + def tag_type(self, obj): return obj.tag.stype - + def get_changelist(self, request, **kwargs): """ Returns the ChangeList class for use on the changelist page. """ return SupertagChangeList - + def changelist_view(self, request, extra_context=None): if request.method == 'POST' and '_update_tags' in request.POST: - ctype_id = request.GET.get(u'content_type__id', [False,]) + ctype_id = request.GET.get(u'content_type__id', [False, ]) obj_id = request.GET.get('object_id', [False]) if ctype_id == False or obj_id == False: return HttpResponseRedirect(request.get_full_path()) @@ -139,7 +141,7 @@ def changelist_view(self, request, extra_context=None): return HttpResponseRedirect(request.get_full_path()) else: return super(SuperTaggedItemAdmin, self).changelist_view(request, extra_context) - + def relevance_bar(self, obj): from django.template import Context from django.template.loader import get_template @@ -147,41 +149,41 @@ def relevance_bar(self, obj): relevance = "%d%%" % (obj.relevance / 10.0) else: relevance = "0" - + tmpl = get_template("admin/supertagging/relevancebar.html") ctxt = Context({'relevance': relevance}) return tmpl.render(ctxt) relevance_bar.allow_tags = True relevance_bar.admin_order_field = 'relevance' relevance_bar.short_description = 'Relevance' - + def get_actions(self, request): return [] class SuperTaggedRelationItemAdmin(admin.ModelAdmin): - list_display = ('content_object', 'relation', 'field', 'process_type', 'item_date') + list_display = ('content_object', 'relation', 'field', 'process_type', 'item_date') list_filter = ('process_type', 'field') - + raw_id_fields = ('relation',) - - + + class SuperTagRelationAdmin(admin.ModelAdmin): list_display = ('tag', 'name', 'stype',) ordering = ('tag', ) search_fields = ('stype', 'name', 'tag__name') list_filter = ('stype', 'name', ) - + raw_id_fields = ('tag',) - - + + class SuperTagProcessQueueAdmin(admin.ModelAdmin): list_display = ('__unicode__', 'locked') actions = [lock_items, unlock_items] - + admin.site.register(SuperTag, SuperTagAdmin) admin.site.register(SuperTaggedItem, SuperTaggedItemAdmin) admin.site.register(SuperTagRelation, SuperTagRelationAdmin) admin.site.register(SuperTaggedRelationItem, SuperTaggedRelationItemAdmin) -admin.site.register(SuperTagProcessQueue, SuperTagProcessQueueAdmin) \ No newline at end of file +admin.site.register(SuperTagProcessQueue, SuperTagProcessQueueAdmin) diff --git a/supertagging/calais.py b/supertagging/calais.py index 86bb5f0..ce71de7 100644 --- a/supertagging/calais.py +++ b/supertagging/calais.py @@ -6,7 +6,9 @@ 1.5 updated by Corey Oordt 8/3/2011 """ -import httplib, urllib, re +import httplib +import urllib +import re import simplejson as json from StringIO import StringIO @@ -19,16 +21,16 @@ __version__ = "1.5" VALID_CONTENT_TYPES = ( - 'text/xml', - 'text/html', - 'text/raw', + 'text/xml', + 'text/html', + 'text/raw', 'text/htmlraw', ) VALID_OUTPUT_TYPES = ( - "xml/rdf", - "text/simple", - "text/microformats", + "xml/rdf", + "text/simple", + "text/microformats", "application/json", "text/n3", ) @@ -36,32 +38,35 @@ VALID_BOOLEANS = ('true', 'false', 't', 'f',) VALID_METADATA_TYPES = ( - "GenericRelations", + "GenericRelations", "SocialTags", "GenericRelations,SocialTags", "SocialTags,GenericRelations", ) + class AppURLopener(urllib.FancyURLopener): - version = "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.5) Gecko/2008121623 Ubuntu/8.10 (intrepid)Firefox/3.0.5" # Lie shamelessly to Wikipedia. + version = "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.5) Gecko/2008121623 Ubuntu/8.10 (intrepid)Firefox/3.0.5" # Lie shamelessly to Wikipedia. urllib._urlopener = AppURLopener() + class Calais(): """ - Python class that knows how to talk to the OpenCalais API. Use the analyze() and analyze_url() methods, which return CalaisResponse objects. + Python class that knows how to talk to the OpenCalais API. Use the + analyze() and analyze_url() methods, which return CalaisResponse objects. """ api_key = None processing_directives = { - "contentType": "TEXT/RAW", - "outputFormat": "application/json", - "reltagBaseURL": '', - "calculateRelevanceScore": True, - "enableMetadataType": '', - "docRDFaccessible": True, + "contentType": "TEXT/RAW", + "outputFormat": "application/json", + "reltagBaseURL": '', + "calculateRelevanceScore": True, + "enableMetadataType": '', + "docRDFaccessible": True, } user_directives = { - "allowDistribution": False, - "allowSearch": False, + "allowDistribution": False, + "allowSearch": False, "externalID": '', "submitter": "python-calais client v.%s" % __version__, } @@ -70,42 +75,49 @@ class Calais(): def __init__(self, api_key, submitter=None): self.api_key = api_key if submitter: - self.user_directives["submitter"]=submitter - + self.user_directives["submitter"] = submitter + def _validate_directives(self): """ Validate that the directives are valid and have valid values """ - if (self.processing_directives['contentType'] and - self.processing_directives['contentType'].lower() not in VALID_CONTENT_TYPES): + if (self.processing_directives['contentType'] and + self.processing_directives['contentType'].lower() + not in VALID_CONTENT_TYPES): raise TypeError("%s is not a valid content type.") if (self.processing_directives['outputFormat'] and - self.processing_directives['outputFormat'].lower() not in VALID_OUTPUT_TYPES): + self.processing_directives['outputFormat'].lower() + not in VALID_OUTPUT_TYPES): raise TypeError("%s is not a valid output type.") if (self.processing_directives['enableMetadataType'] and - self.processing_directives['enableMetadataType'].lower() not in VALID_METADATA_TYPES): + self.processing_directives['enableMetadataType'].lower() + not in VALID_METADATA_TYPES): raise TypeError("%s is not a valid metadata type.") - - if (self.processing_directives['calculateRelevanceScore'] and + + if (self.processing_directives['calculateRelevanceScore'] and not isinstance(self.processing_directives['calculateRelevanceScore'], bool)): - if self.processing_directives['calculateRelevanceScore'].lower() not in VALID_BOOLEAN_TYPES: + if (self.processing_directives['calculateRelevanceScore'].lower() + not in VALID_BOOLEANS): raise TypeError("%s is not a valid boolean type.") - - if (self.processing_directives['docRDFaccessible'] and + + if (self.processing_directives['docRDFaccessible'] and not isinstance(self.processing_directives['docRDFaccessible'], bool)): - if self.processing_directives['docRDFaccessible'].lower() not in VALID_BOOLEAN_TYPES: + if (self.processing_directives['docRDFaccessible'].lower() + not in VALID_BOOLEANS): raise TypeError("%s is not a valid boolean type.") - - if (self.user_directives['allowDistribution'] and + + if (self.user_directives['allowDistribution'] and not isinstance(self.user_directives['allowDistribution'], bool)): - if self.user_directives['allowDistribution'].lower() not in VALID_BOOLEAN_TYPES: + if (self.user_directives['allowDistribution'].lower() + not in VALID_BOOLEANS): raise TypeError("%s is not a valid boolean type.") - - if (self.user_directives['allowSearch'] and + + if (self.user_directives['allowSearch'] and not isinstance(self.user_directives['allowSearch'], bool)): - if self.user_directives['allowSearch'].lower() not in VALID_BOOLEAN_TYPES: + if (self.user_directives['allowSearch'].lower() + not in VALID_BOOLEANS): raise TypeError("%s is not a valid boolean type.") - + def _get_param_headers(self): headers = {} for key, val in self.processing_directives.items(): @@ -122,7 +134,7 @@ def _get_param_headers(self): if val: headers[key] = val return headers - + def rest_POST(self, content): headers = { "x-calais-licenseID": self.api_key, @@ -137,7 +149,7 @@ def rest_POST(self, content): def get_random_id(self): """ - Creates a random 10-character ID for your submission. + Creates a random 10-character ID for your submission. """ import string from random import choice @@ -149,7 +161,7 @@ def get_random_id(self): def get_content_id(self, text): """ - Creates a SHA1 hash of the text of your submission. + Creates a SHA1 hash of the text of your submission. """ import hashlib h = hashlib.sha1() @@ -162,9 +174,9 @@ def preprocess_html(self, html): return html def analyze(self, content, content_type="TEXT/RAW", external_id=None): - if not (content and len(content.strip())): + if not (content and len(content.strip())): return None - self.processing_directives["contentType"]=content_type + self.processing_directives["contentType"] = content_type if external_id: self.user_directives["externalID"] = external_id return CalaisResponse(self.rest_POST(content)) @@ -181,7 +193,7 @@ def analyze_file(self, fn): except: raise ValueError("Can not determine file type for '%s'" % fn) if filetype == "text/plain": - content_type="TEXT/RAW" + content_type = "TEXT/RAW" f = open(fn) content = f.read() f.close() @@ -191,16 +203,18 @@ def analyze_file(self, fn): content = self.preprocess_html(f.read()) f.close() else: - raise ValueError("Only plaintext and HTML files are currently supported. ") + raise ValueError("Only plaintext and HTML files are currently supported.") return self.analyze(content, content_type=content_type, external_id=fn) + class CalaisResponse(): """ - Encapsulates a parsed Calais response and provides easy pythonic access to the data. + Encapsulates a parsed Calais response and provides easy pythonic access + to the data. """ raw_response = None simplified_response = None - + def __init__(self, raw_result): try: self.raw_response = json.load(StringIO(raw_result)) @@ -208,21 +222,21 @@ def __init__(self, raw_result): raise ValueError(raw_result) self.simplified_response = self._simplify_json(self.raw_response) self.__dict__['doc'] = self.raw_response['doc'] - for k,v in self.simplified_response.items(): + for k, v in self.simplified_response.items(): self.__dict__[k] = v - + def _simplify_json(self, json): result = {} # First, resolve references for element in json.values(): - for k,v in element.items(): - if isinstance(v, unicode) and v.startswith("http://") and json.has_key(v): + for k, v in element.items(): + if isinstance(v, unicode) and v.startswith("http://") and v in json: element[k] = json[v] for k, v in json.items(): - if v.has_key("_typeGroup"): + if "_typeGroup" in v: group = v["_typeGroup"] - if not result.has_key(group): - result[group]=[] + if group not in result: + result[group] = [] del v["_typeGroup"] v["__reference"] = k result[group].append(v) @@ -233,13 +247,13 @@ def print_summary(self): return None info = self.doc['info'] print "Calais Request ID: %s" % info['calaisRequestID'] - if info.has_key('externalID'): + if 'externalID' in info: print "External ID: %s" % info['externalID'] - if info.has_key('docTitle'): + if 'docTitle' in info: print "Title: %s " % info['docTitle'] print "Language: %s" % self.doc['meta']['language'] print "Extractions: " - for k,v in self.simplified_response.items(): + for k, v in self.simplified_response.items(): print "\t%d %s" % (len(v), k) def print_entities(self): @@ -259,9 +273,9 @@ def print_relations(self): return None for relation in self.relations: print relation['_type'] - for k,v in relation.items(): + for k, v in relation.items(): if not k.startswith("_"): if isinstance(v, unicode): - print "\t%s:%s" % (k,v) - elif isinstance(v, dict) and v.has_key('name'): + print "\t%s:%s" % (k, v) + elif isinstance(v, dict) and 'name' in v: print "\t%s:%s" % (k, v['name']) diff --git a/supertagging/fields.py b/supertagging/fields.py index 18295ee..aeb4007 100644 --- a/supertagging/fields.py +++ b/supertagging/fields.py @@ -33,6 +33,7 @@ DEFAULT_PROTOCOL = 2 + class PickledObject(str): """ A subclass of string so it can be told whether a string is a pickled @@ -177,4 +178,4 @@ def get_db_prep_lookup(self, lookup_type, value, connection=None, prepared=False except ImportError: pass else: - add_introspection_rules([], [r"^supertagging\.fields\.PickledObjectField"]) \ No newline at end of file + add_introspection_rules([], [r"^supertagging\.fields\.PickledObjectField"]) diff --git a/supertagging/handlers.py b/supertagging/handlers.py index f117278..8ac4c3a 100644 --- a/supertagging/handlers.py +++ b/supertagging/handlers.py @@ -1,17 +1,20 @@ from django.db.models import get_model from django.db.models.signals import post_save, post_delete -from supertagging.settings import USE_QUEUE, MODULES, AUTO_PROCESS, ST_DEBUG, MARKUP, MARKUP_FIELD_SUFFIX, REGISTER_MODELS +from supertagging.settings import (USE_QUEUE, MODULES, AUTO_PROCESS, ST_DEBUG, + MARKUP, MARKUP_FIELD_SUFFIX, REGISTER_MODELS) from supertagging import register + def save_handler(sender, instance, **kwargs): if instance: - from supertagging.modules import process, add_to_queue + from supertagging.modules import process, add_to_queue if USE_QUEUE: add_to_queue(instance) else: process(instance) + def delete_handler(sender, instance, **kwargs): if instance: from supertagging.modules import clean_up, remove_from_queue @@ -20,22 +23,23 @@ def delete_handler(sender, instance, **kwargs): else: clean_up(instance) + def setup_handlers(): try: - for k,v in MODULES.items(): + for k, v in MODULES.items(): app_label, model_name = k.split('.') model = get_model(app_label, model_name) - + # Add a tag attribute to the instance. if REGISTER_MODELS: register(model) - + # Setup post save and post delete handlers if model exists if model and AUTO_PROCESS: post_save.connect(save_handler, sender=model) post_delete.connect(delete_handler, sender=model) - - if MARKUP: + + if MARKUP: # Add a custom attribute to a model with a marked up # version of the field specified. for f in v.get('fields', []): @@ -46,6 +50,7 @@ def setup_handlers(): handler = get_handler_module(f.get('markup_handler', None)) nfield = "%s__%s" % (field, MARKUP_FIELD_SUFFIX) setattr(model, nfield, handler(model, field)) - + except Exception, e: - if ST_DEBUG: raise Exception(e) + if ST_DEBUG: + raise Exception(e) diff --git a/supertagging/management/commands/st_migrate_pickled_object_fields.py b/supertagging/management/commands/st_migrate_pickled_object_fields.py index 759da49..cc1213d 100644 --- a/supertagging/management/commands/st_migrate_pickled_object_fields.py +++ b/supertagging/management/commands/st_migrate_pickled_object_fields.py @@ -1,18 +1,20 @@ #!/usr/bin/python -import time + from django.core.management.base import BaseCommand from django.db import transaction, connection from supertagging.models import SuperTag, SuperTagRelation, SuperTaggedItem, SuperTaggedRelationItem + try: import cPickle as pickle except ImportError: import pickle - + + class Command(BaseCommand): def handle(self, *args, **kwargs): c = Core() c.execute() - + def ResultIter(cursor, arraysize=1000): 'An iterator that uses fetchmany to keep memory usage down' @@ -22,25 +24,25 @@ def ResultIter(cursor, arraysize=1000): break for result in results: yield result - + def convert_fields(model, field): print "-- Processing: %s" % model cursor = connection.cursor() - + sql = "SELECT id, %s FROM %s ORDER BY id LIMIT 100" % (field, model._meta.db_table) cursor.execute(sql) - + for result in ResultIter(cursor): pid, value = result print "-- Fixing row: %s" % pid - + try: old_data = pickle.loads(str(value)) except Exception, e: print "-- Error loading old data, continuing.. (%s)" % e old_data = value - + try: item = model.objects.get(pk=pid) if hasattr(item, field): @@ -48,27 +50,28 @@ def convert_fields(model, field): item.save() except Exception, e: print "-- Error occuring while saving %s: %s" % (model, e) - + print "-- Done Processing: %s" % model + class Core(object): """ Convert the old Pickled data - """ + """ @transaction.commit_manually def execute(self): print "Begin Pickled Field Conversion" - + convert_fields(SuperTag, "properties") transaction.commit() - + convert_fields(SuperTagRelation, "properties") transaction.commit() - + convert_fields(SuperTaggedItem, "instances") transaction.commit() - + convert_fields(SuperTaggedRelationItem, "instances") transaction.commit() - + print "Done" diff --git a/supertagging/management/commands/st_process_queue.py b/supertagging/management/commands/st_process_queue.py index 43e6a0f..5504f5a 100644 --- a/supertagging/management/commands/st_process_queue.py +++ b/supertagging/management/commands/st_process_queue.py @@ -2,12 +2,10 @@ import time from django.core.management.base import BaseCommand from django.db import transaction -from django.conf import settings -from django.contrib.contenttypes.models import ContentType from supertagging.models import SuperTagProcessQueue from supertagging.modules import process -from supertagging import settings as st_settings + class Command(BaseCommand): def handle(self, *args, **kwargs): @@ -18,7 +16,7 @@ def handle(self, *args, **kwargs): class Core(object): """ The Core is responsible for driving Supertagging - """ + """ @transaction.commit_manually def execute(self): """ @@ -50,7 +48,7 @@ def execute(self): print 'Done' failed += 1 time.sleep(1) - + print 'Unlocking objects...' SuperTagProcessQueue.objects.filter(pk__in=objs_to_reset).update(locked=False) transaction.commit() @@ -59,5 +57,5 @@ def execute(self): SuperTagProcessQueue.objects.filter(pk__in=objs_to_del).delete() transaction.commit() print 'Done' - print '%s of %s objects processed. %s failed.' % (processed, + print '%s of %s objects processed. %s failed.' % (processed, processed + failed, failed) diff --git a/supertagging/managers.py b/supertagging/managers.py index de5a5fd..d71e132 100644 --- a/supertagging/managers.py +++ b/supertagging/managers.py @@ -6,6 +6,7 @@ from django.db import models from supertagging.models import SuperTag, SuperTaggedItem + class ModelTagManager(models.Manager): """ A manager for retrieving tags for a particular model. @@ -25,6 +26,7 @@ def related(self, tags, *args, **kwargs): def usage(self, *args, **kwargs): return SuperTag.objects.usage_for_model(self.model, *args, **kwargs) + class ModelTaggedItemManager(models.Manager): """ A manager for retrieving model instances based on their tags. @@ -47,6 +49,7 @@ def with_any(self, tags, queryset=None): else: return SuperTaggedItem.objects.get_union_by_model(queryset, tags) + class TagDescriptor(object): """ A descriptor which provides access to a ``ModelTagManager`` for @@ -62,8 +65,8 @@ def __get__(self, instance, owner): return SuperTag.objects.get_for_object(instance) def __set__(self, instance, value): - # Cannot update tags + # Cannot update tags raise NotImplementedError def __delete__(self, instance): - raise NotImplementedError \ No newline at end of file + raise NotImplementedError diff --git a/supertagging/markup.py b/supertagging/markup.py index 873dd43..4728ecb 100644 --- a/supertagging/markup.py +++ b/supertagging/markup.py @@ -1,11 +1,11 @@ from django.contrib.contenttypes.models import ContentType -from django.template.loader import render_to_string, get_template +from django.template.loader import render_to_string from django.core.cache import cache -from django.db.models import get_model from supertagging import settings from supertagging.models import SuperTaggedItem + class MarkupHandler(object): """ Default Markup handler @@ -14,53 +14,55 @@ def __init__(self, model, field): self.field = field self.model = model self.content_type = ContentType.objects.get_for_model(model) - + def __get__(self, instance, owner): if not instance: return - + data = self._get_cached_value(instance) if data: return data - + try: data = self.handle(instance) except Exception, e: data = getattr(instance, self.field) - if settings.ST_DEBUG: raise Exception(e) - + if settings.ST_DEBUG: + raise Exception(e) + cache.set(self._get_cache_key(instance), data, settings.MARKUP_CONTENT_CACHE_TIMEOUT) return data - + def _get_cache_key(self, instance=None): if instance: return "ST_HANDLER.%s.%s.%s" % (self.content_type.pk, instance.pk, self.field) return None - + def _get_cached_value(self, instance=None): if instance and self._get_cache_key(instance): key = self._get_cache_key(instance) return cache.get(key) return None - + def handle(self, instance=None): if instance: return markup_content(instance, self.field) return "" - - + + def invalidate_markup_cache(obj, field): if not obj: return - + ctype = ContentType.objects.get_for_model(obj) key = "ST_HANDLER.%s.%s.%s" % (ctype.pk, obj.pk, field) cache.delete(key) - + + def get_handler_module(module): if not module: return MarkupHandler - + mod, f, i, fn = None, None, None, None msplit = module.split(".") if len(msplit) == 1: @@ -69,7 +71,7 @@ def get_handler_module(module): f = msplit[:-1] i = msplit[-1:] fn = ".".join(f) - + if mod: try: return __import__(mod) @@ -81,28 +83,32 @@ def get_handler_module(module): return getattr(mod, i[0]) except: return MarkupHandler - + + class FailedMarkupValidation(Exception): - def __init__(self, value): - self.parameter = value - def __str__(self): - return repr(self.parameter) + def __init__(self, value): + self.parameter = value + + def __str__(self): + return repr(self.parameter) + def tag_instance_cmp(x, y): if isinstance(x, dict) and isinstance(y, dict): - return cmp(x['offset'],y['offset']) + return cmp(x['offset'], y['offset']) return cmp(1, 1) + def markup_content(obj, field, markup_template='supertagging/markup.html'): """ - Takes all the items (SuperTaggedItems), and retrieves all the 'instances' to + Takes all the items (SuperTaggedItems), and retrieves all the 'instances' to embed the markup_template. """ ctype = ContentType.objects.get_for_model(obj) items = SuperTaggedItem.objects.filter( - content_type__pk=ctype.pk, object_id=obj.pk, + content_type__pk=ctype.pk, object_id=obj.pk, relevance__gte=settings.MIN_RELEVANCE_MARKUP).select_related() - + value = getattr(obj, field, '') full = [] for item in items: @@ -117,13 +123,13 @@ def markup_content(obj, field, markup_template='supertagging/markup.html'): continue if isinstance(v, dict): v['supertag'] = item.tag - + if not skip: full.extend(i) # Sort the list by the inner dict offset value in reverse full.sort(tag_instance_cmp, reverse=True) - + for n, i in enumerate(full): if 'offset' in i and 'length' in i and 'exact' in i: off, le, act_val = i['offset'], i['length'], i['exact'] @@ -131,29 +137,30 @@ def markup_content(obj, field, markup_template='supertagging/markup.html'): continue else: continue - + # This tests to make sure the next tag does # not overlap the current tag if n != 0: - if 'offset' in full[n-1]: - prev_off = full[n-1]['offset'] - if ((off+1)+le) > prev_off: + if 'offset' in full[n - 1]: + prev_off = full[n - 1]['offset'] + if ((off + 1) + le) > prev_off: continue - + # Validate that the data matches the data returned by calais - if not value[off:(off+le)] == act_val: + if not value[off:(off + le)] == act_val: raise FailedMarkupValidation( - "Markup failed validation: Offset: %s: \"%s\" didn't match \"%s\"" % (off, value[off:(off+le)], act_val)) + "Markup failed validation: Offset: %s: \"%s\" didn't match \"%s\"" + % (off, value[off:(off + le)], act_val)) break - + tag = i['supertag'] val = render_to_string(markup_template, {'tag': tag, 'actual_value': act_val}) - pre, suf, repl = '','','' - + pre, suf, repl = '', '', '' + pre = value[:off] - suf = value[(off+le):] - repl = value[off:(off+le)] - - value = pre+val+suf - - return value \ No newline at end of file + suf = value[(off + le):] + repl = value[off:(off + le)] + + value = pre + val + suf + + return value diff --git a/supertagging/models.py b/supertagging/models.py index afe0f49..713f544 100644 --- a/supertagging/models.py +++ b/supertagging/models.py @@ -7,8 +7,8 @@ from supertagging.handlers import setup_handlers from supertagging.fields import PickledObjectField -from supertagging.utils import (calculate_cloud, get_tag_list, - get_queryset_and_model, LOGARITHMIC, render_item, +from supertagging.utils import (calculate_cloud, get_tag_list, + get_queryset_and_model, LOGARITHMIC, render_item, retrieve_freebase_name, retrieve_freebase_desc) from supertagging import settings as st_settings @@ -17,7 +17,8 @@ ################### ## MANAGERS ## ################### - + + class SuperTagManager(models.Manager): def get_by_name(self, **kwargs): """ @@ -28,15 +29,15 @@ def get_by_name(self, **kwargs): obj = super(SuperTagManager, self).get(**kwargs) # If object has a substitute speficied, use that tag enstead obj = obj.substitute or obj - + # Return object if freebase is not used if not st_settings.USE_FREEBASE: return obj - + # Try to find the name using freebase fb_name = retrieve_freebase_name(obj.name, obj.stype) try: - # Try to retrieve the existing name given by freebase + # Try to retrieve the existing name given by freebase # in our database new_tag = self.get(name__iexact=fb_name) # Return the new tag or the new tags substitute @@ -44,20 +45,20 @@ def get_by_name(self, **kwargs): except: # Simply return the obj if something went wrong return obj - + def create_alternate(self, **kwargs): """ - Alternate method for creating SuperTags while optionally using + Alternate method for creating SuperTags while optionally using freebase to disambiguate the names """ # Retrieve the arguments used to create a new tag name = kwargs.get("name", None) stype = kwargs.get("stype", None) - + # Create and return the new tag if freebase is not used. if not (st_settings.USE_FREEBASE and name): return super(SuperTagManager, self).create(**kwargs) - + fb_name = retrieve_freebase_name(name, stype) try: new_tag = self.get(name__iexact=fb_name) @@ -65,12 +66,12 @@ def create_alternate(self, **kwargs): except: kwargs["name"] = fb_name.lower() kwargs["slug"] = slugify(fb_name)[:50] - + return super(SuperTagManager, self).create(**kwargs) def get_for_object(self, obj, **kwargs): """ - Returns tags for an object, also returns the relevance score from + Returns tags for an object, also returns the relevance score from the supertaggingitem table. """ ctype = ContentType.objects.get_for_model(obj) @@ -78,31 +79,31 @@ def get_for_object(self, obj, **kwargs): if 'field' in kwargs: extra_args['supertaggeditem__field'] = kwargs['field'] order_by = kwargs.get('order_by', '-relevance') - + return self.filter( supertaggeditem__content_type__pk=ctype.pk, supertaggeditem__object_id=obj.pk, **extra_args).annotate( relevance=models.Max('supertaggeditem__relevance') ).order_by(order_by) - + def get_topics_for_object(self, obj): ctype = ContentType.objects.get_for_model(obj) ids = self.filter(supertaggeditem__content_type__pk=ctype.pk, - supertaggeditem__object_id=obj.pk, + supertaggeditem__object_id=obj.pk, stype='Topic').values('id') - + return self.filter(id__in=ids) - - def _get_usage(self, model, counts=False, min_count=None, + + def _get_usage(self, model, counts=False, min_count=None, extra_joins=None, extra_criteria=None, params=None): """ Perform the custom SQL query for ``usage_for_model`` and ``usage_for_queryset``. """ - if min_count is not None: + if min_count is not None: counts = True - + model_table = qn(model._meta.db_table) model_pk = '%s.%s' % (model_table, qn(model._meta.pk.column)) query = """ @@ -115,7 +116,7 @@ def _get_usage(self, model, counts=False, min_count=None, ON %(tagged_item)s.object_id = %(model_pk)s %%s WHERE %(tagged_item)s.content_type_id = %(content_type_id)s - %%s + %%s GROUP BY %(tag)s.id, %(tag)s.name, %(tag)s.slug %%s ORDER BY %(tag)s.name ASC""" % { @@ -126,12 +127,12 @@ def _get_usage(self, model, counts=False, min_count=None, 'model_pk': model_pk, 'content_type_id': ContentType.objects.get_for_model(model).pk, } - + min_count_sql = '' if min_count is not None: min_count_sql = 'HAVING COUNT(%s) >= %%s' % model_pk params.append(min_count) - + cursor = connection.cursor() cursor.execute(query % (extra_joins, extra_criteria, min_count_sql), params) tags = [] @@ -141,20 +142,20 @@ def _get_usage(self, model, counts=False, min_count=None, t.count = row[2] tags.append(t) return tags - + def usage_for_model(self, model, counts=False, min_count=None, filters=None): """ Obtain a list of tags associated with instances of the given Model class. - + If ``counts`` is True, a ``count`` attribute will be added to each tag, indicating how many times it has been used against the Model class in question. - + If ``min_count`` is given, only tags which have a ``count`` greater than or equal to ``min_count`` will be returned. Passing a value for ``min_count`` implies ``counts=True``. - + To limit the tags (and counts, if specified) returned to those used by a subset of the Model's instances, pass a dictionary of field lookups to be applied to the given Model as the @@ -162,28 +163,28 @@ def usage_for_model(self, model, counts=False, min_count=None, filters=None): """ if filters is None: filters = {} - + queryset = model._default_manager.filter() for f in filters.items(): queryset.query.add_filter(f) usage = self.usage_for_queryset(queryset, counts, min_count) - + return usage - + def usage_for_queryset(self, queryset, counts=False, min_count=None): """ Obtain a list of tags associated with instances of a model contained in the given queryset. - + If ``counts`` is True, a ``count`` attribute will be added to each tag, indicating how many times it has been used against the Model class in question. - + If ``min_count`` is given, only tags which have a ``count`` greater than or equal to ``min_count`` will be returned. Passing a value for ``min_count`` implies ``counts=True``. """ - + if getattr(queryset.query, 'get_compiler', None): # Django 1.2 and up compatible (multiple databases) compiler = queryset.query.get_compiler(using='default') @@ -193,7 +194,7 @@ def usage_for_queryset(self, queryset, counts=False, min_count=None): # Django 1.1 and down compatible (single database) extra_joins = ' '.join(queryset.query.get_from_clause()[0][1:]) where, params = queryset.query.where.as_sql() - + # extra_joins = ' '.join( # queryset.query.get_compiler(using='default').get_from_clause()[0][1:]) # where, params = queryset.query.where.as_sql() @@ -201,9 +202,9 @@ def usage_for_queryset(self, queryset, counts=False, min_count=None): extra_criteria = 'AND %s' % where else: extra_criteria = '' - return self._get_usage(queryset.model, counts, min_count, + return self._get_usage(queryset.model, counts, min_count, extra_joins, extra_criteria, params) - + def cloud_for_model(self, model, steps=4, distribution=LOGARITHMIC, filters=None, min_count=None): """ @@ -211,20 +212,20 @@ def cloud_for_model(self, model, steps=4, distribution=LOGARITHMIC, Model, giving each tag a ``count`` attribute indicating how many times it has been used and a ``font_size`` attribute for use in displaying a tag cloud. - + ``steps`` defines the range of font sizes - ``font_size`` will be an integer between 1 and ``steps`` (inclusive). - + ``distribution`` defines the type of font size distribution algorithm which will be used - logarithmic or linear. It must be either ``supertagging.utils.LOGARITHMIC`` or ``supertagging.utils.LINEAR``. - + To limit the tags displayed in the cloud to those associated with a subset of the Model's instances, pass a dictionary of field lookups to be applied to the given Model as the ``filters`` argument. - + To limit the tags displayed in the cloud to those with a ``count`` greater than or equal to ``min_count``, pass a value for the ``min_count`` argument. @@ -237,9 +238,9 @@ def cloud_for_model(self, model, steps=4, distribution=LOGARITHMIC, class SuperTagRelationManager(models.Manager): def get_for_tag(self, tag, **kwargs): return self.filter( - tag__pk=tag.id, - tag__enabled=True, - supertaggedrelationitem__item_date__isnull=False, + tag__pk=tag.id, + tag__enabled=True, + supertaggedrelationitem__item_date__isnull=False, **kwargs).distinct().order_by('-supertaggedrelationitem__item_date') @@ -249,7 +250,7 @@ def active(self): Return tagged items that aren't ignored. """ return self.get_query_set().filter(ignore=False) - + def get_by_model(self, queryset_or_model, tags): """ Create a ``QuerySet`` containing instances of the specified @@ -267,11 +268,11 @@ def get_by_model(self, queryset_or_model, tags): tag = tags[0] else: return self.get_intersection_by_model(queryset_or_model, tags) - + content_type = ContentType.objects.get_for_model(model) opts = self.model._meta tagged_item_table = qn(opts.db_table) - + return queryset.active().extra( tables=[opts.db_table], where=[ @@ -283,7 +284,7 @@ def get_by_model(self, queryset_or_model, tags): ], params=[content_type.pk, tag.pk], ) - + def get_intersection_by_model(self, queryset_or_model, tags): """ Create a ``QuerySet`` containing instances of the specified @@ -292,10 +293,10 @@ def get_intersection_by_model(self, queryset_or_model, tags): tags = get_tag_list(tags) tag_count = len(tags) queryset, model = get_queryset_and_model(queryset_or_model) - + if not tag_count: return model._default_manager.none() - + model_table = qn(model._meta.db_table) # This query selects the ids of all objects which have all the # given tags. @@ -314,7 +315,7 @@ def get_intersection_by_model(self, queryset_or_model, tags): 'tag_id_placeholders': ','.join(['%s'] * tag_count), 'tag_count': tag_count, } - + cursor = connection.cursor() cursor.execute(query, [tag.pk for tag in tags]) object_ids = [row[0] for row in cursor.fetchall()] @@ -322,7 +323,7 @@ def get_intersection_by_model(self, queryset_or_model, tags): return queryset.filter(pk__in=object_ids) else: return model._default_manager.none() - + def get_union_by_model(self, queryset_or_model, tags): """ Create a ``QuerySet`` containing instances of the specified @@ -429,10 +430,10 @@ class SuperTaggedRelationItemManager(models.Manager): def get_for_object(self, obj, **kwargs): ctype = ContentType.objects.get_for_model(obj) return self.filter(content_type__pk=ctype.pk, object_id=obj.pk, **kwargs) - + def get_for_tag_in_object(self, tag, obj): ctype = ContentType.objects.get_for_model(obj) - return self.filter(relation__tag__pk=tag.pk, + return self.filter(relation__tag__pk=tag.pk, content_type__pk=ctype.pk, object_id=obj.pk) @@ -441,7 +442,7 @@ def get_for_tag_in_object(self, tag, obj): ################### class SuperTag(models.Model): calais_id = models.CharField(_("Calais ID"), max_length=255, unique=True) - substitute = models.ForeignKey("self", null=True, blank=True, + substitute = models.ForeignKey("self", null=True, blank=True, related_name="substitute_tagsubstitute", verbose_name=_("Substitute"), help_text=_("""Tag to use instead of this one. This will also update @@ -450,67 +451,67 @@ class SuperTag(models.Model): slug = models.SlugField(_("Slug"), max_length=150) stype = models.CharField(_("Type"), max_length=100) properties = PickledObjectField(_("Properties"), null=True, blank=True) - enabled = models.BooleanField(_("Enabled"), default=True, - help_text=_("""If False, will cause the tag to be disabled + enabled = models.BooleanField(_("Enabled"), default=True, + help_text=_("""If False, will cause the tag to be disabled and all assoicated items and relation items will be removed.""")) - + if st_settings.INCLUDE_DISPLAY_FIELDS: # Extra fields that optionally can be used for display. from django.core.files.storage import get_storage_class IMAGE_STORAGE = get_storage_class(st_settings.DEFAULT_STORAGE) # Create the display fields - display_name = models.CharField(_("Display Name"), max_length=150, + display_name = models.CharField(_("Display Name"), max_length=150, null=True, blank=True) - description = models.TextField(_("Description"), blank=True, + description = models.TextField(_("Description"), blank=True, null=True, help_text=_('Tag Description.')) - icon = models.ImageField(_("Icon"), upload_to="supertagging/icons/", - blank=True, null=True, storage=IMAGE_STORAGE(), + icon = models.ImageField(_("Icon"), upload_to="supertagging/icons/", + blank=True, null=True, storage=IMAGE_STORAGE(), help_text=_('Tag Icon.')) related = models.ManyToManyField("self", blank=True, null=True, - verbose_name=_("Related Tags"), + verbose_name=_("Related Tags"), help_text=_("Assign related tags manually.")) objects = SuperTagManager() def __unicode__(self): return "%s - %s" % (self.name, self.stype) - + def get_name(self): if self.has_display_fields(): return self.display_name or self.name return self.name - + def has_display_fields(self): if st_settings.INCLUDE_DISPLAY_FIELDS: return True return False - + def render(self, template=None, suffix=None): return render_item(None, self.stype, template, suffix, template_path="supertagging/render/tags", context={'obj': self}) - + class Meta: ordering = ('name',) - - def save(self, *args, **kwargs): + + def save(self, *args, **kwargs): super(SuperTag, self).save(*args, **kwargs) # If display fields are available and FREEBASE_RETRIEVE_DESCRIPTIONS is True # and the description field is empty, try to get a description from Freebase if self.has_display_fields() and st_settings.FREEBASE_RETRIEVE_DESCRIPTIONS and not self.description: self.description = retrieve_freebase_desc(self.name, self.stype) - - # If tag is set to be disabled and REMOVE_REL_ON_DISABLE is True, + + # If tag is set to be disabled and REMOVE_REL_ON_DISABLE is True, # remove all Tagged Items and Tagged Relation Items if not self.enabled and st_settings.REMOVE_REL_ON_DISABLE: SuperTaggedItem.objects.filter(tag__pk=self.pk).delete() SuperTaggedRelationItem.objects.filter(relation__tag__pk=self.pk).delete() - - # If a substitute is supplied, change all SuperTaggedItem's and + + # If a substitute is supplied, change all SuperTaggedItem's and # SuperTagRelation's to have this new tag if self.substitute and st_settings.SUBSTITUTE_TAG_UPDATE: items = self.supertaggeditem_set.all() - relations = self.supertagrelation_set.all() + relations = self.supertagrelation_set.all() items.update(tag=self.substitute) relations.update(tag=self.substitute) @@ -525,7 +526,7 @@ class SuperTagRelation(models.Model): def __unicode__(self): return "%s - %s - %s" % (self.stype, self.tag.name, self.name) - + def render(self, template=None, suffix=None): return render_item(None, self.stype, template, suffix, template_path="supertagging/render/relations", @@ -543,20 +544,21 @@ class SuperTaggedItem(models.Model): instances = PickledObjectField(null=True, blank=True) item_date = models.DateTimeField(null=True, blank=True) ignore = models.BooleanField(default=False) - + objects = SuperTaggedItemManager() - + class Meta: ordering = ('-relevance',) - + def __unicode__(self): return u'%s of %s' % (self.tag, unicode(self.content_object)) - + def render(self, template=None, suffix=None): return render_item(self, None, template, suffix, - template_path="supertagging/render/tagged_items", + template_path="supertagging/render/tagged_items", context={'obj': self.content_object, 'content': self}) + class SuperTaggedRelationItem(models.Model): relation = models.ForeignKey(SuperTagRelation) content_type = models.ForeignKey(ContentType) @@ -565,41 +567,43 @@ class SuperTaggedRelationItem(models.Model): field = models.CharField(max_length=100) process_type = models.CharField(max_length=20, null=True, blank=True) instances = PickledObjectField(null=True, blank=True) - + item_date = models.DateTimeField(null=True, blank=True) - + objects = SuperTaggedRelationItemManager() - + class Meta: ordering = ['-item_date'] - + def __unicode__(self): return "%s of %s" % (self.relation.name, str(self.content_object)) - + def render(self, template=None, suffix=None): return render_item(self, None, template, suffix, template_path="supertagging/render/tagged_relations", context={'obj': self.content_object, 'content': self}) + class SuperTagProcessQueue(models.Model): content_type = models.ForeignKey(ContentType) object_id = models.PositiveIntegerField() content_object = generic.GenericForeignKey('content_type', 'object_id') locked = models.BooleanField(default=False) - + def __unicode__(self): return 'Queue Item: <%s> %s' % ( - self.content_type, + self.content_type, unicode(str(self.content_object), 'utf-8')) - + class Meta: verbose_name = "Process Queue" verbose_name_plural = "Process Queue" + def _clean_tagged_relation_items(sender, instance, **kwargs): if not instance: return - + items = SuperTaggedRelationItem.objects.filter( relation__tag__pk=instance.tag.pk, content_type__pk=instance.content_type.pk, diff --git a/supertagging/modules/__init__.py b/supertagging/modules/__init__.py index 009ad32..de152b3 100644 --- a/supertagging/modules/__init__.py +++ b/supertagging/modules/__init__.py @@ -1,7 +1,8 @@ """ Django-SuperTagging """ -import re, datetime +import re +import datetime from django.contrib.contenttypes.models import ContentType from django.template.defaultfilters import slugify from django.utils.encoding import force_unicode @@ -9,17 +10,19 @@ from supertagging import settings from supertagging.calais import Calais -from supertagging.models import SuperTag, SuperTagRelation, SuperTaggedItem, SuperTaggedRelationItem, SuperTagProcessQueue +from supertagging.models import (SuperTag, SuperTagRelation, SuperTaggedItem, + SuperTaggedRelationItem, SuperTagProcessQueue) from supertagging.markup import invalidate_markup_cache REF_REGEX = "^http://d.opencalais.com/(?P.*)$" + def add_to_queue(instance): """ Add object to the queue. """ # Only add the queue if supplied match_kwargs returns the instance - params = settings.MODULES.get('%s.%s' % (instance._meta.app_label, + params = settings.MODULES.get('%s.%s' % (instance._meta.app_label, instance._meta.module_name), {}) if 'match_kwargs' in params: model = get_model(instance._meta.app_label, instance._meta.module_name) @@ -28,16 +31,17 @@ def add_to_queue(instance): instance = model.objects.get(pk=instance.pk, **params['match_kwargs']) except model.DoesNotExist: return - + cont_type = ContentType.objects.get_for_model(instance) - # If ONLY_NON_TAGGED_OBJECTS is True and 'instance' has been + # If ONLY_NON_TAGGED_OBJECTS is True and 'instance' has been # tagged, DO NOT add to queue if settings.ONLY_NON_TAGGED_OBJECTS and SuperTaggedItem.objects.filter( content_type__pk=cont_type.pk, object_id=str(instance.pk)).count() > 0: return SuperTagProcessQueue.objects.get_or_create( content_type=cont_type, object_id=instance.pk) - + + def remove_from_queue(instance): """ Remove object from the queue. @@ -48,12 +52,13 @@ def remove_from_queue(instance): except SuperTagProcessQueue.DoesNotExist: pass + def process(obj, tags=[]): """ Process the data. """ # In the case when we want to turn off ALL processing of data, while - # preserving AUTO_PROCESS + # preserving AUTO_PROCESS if not settings.ENABLED: return @@ -63,36 +68,36 @@ def process(obj, tags=[]): return try: - params = settings.MODULES['%s.%s' % (obj._meta.app_label, + params = settings.MODULES['%s.%s' % (obj._meta.app_label, obj._meta.module_name)] model = get_model(obj._meta.app_label, obj._meta.module_name) except KeyError, e: if settings.ST_DEBUG: raise KeyError(e) return - + # If no fields are found, nothing can be processed. if not 'fields' in params: if settings.ST_DEBUG: raise Exception('No "fields" found.') else: return - + if 'match_kwargs' in params: try: # Make sure this obj matches the match kwargs obj = model.objects.get(pk=obj.pk, **params['match_kwargs']) except model.DoesNotExist: return - + # Retrieve the default process type process_type = settings.DEFAULT_PROCESS_TYPE - - # If the contentType key is specified in the PROCESSING_DIR set the + + # If the contentType key is specified in the PROCESSING_DIR set the # process type if 'contentType' in settings.PROCESSING_DIR: process_type = settings.PROCESSING_DIR['contentType'] - + # If the MODULES setting specifies a process type set the process type. if 'process_type' in params: process_type = params['process_type'] @@ -106,8 +111,8 @@ def process(obj, tags=[]): c.processing_directives['contentType'] = process_type # Get the object's date. - # First look for the key 'date_field' in the MODULES setting, then - # get_latest_by in the meta class, then check the ordering attribute + # First look for the key 'date_field' in the MODULES setting, then + # get_latest_by in the meta class, then check the ordering attribute # in the meta class. date = None date_fields = [] @@ -117,103 +122,111 @@ def process(obj, tags=[]): date_fields.append(obj._meta.get_latest_by) else: date_fields = obj._meta.ordering - + # Retrieve the Django content type for the obj ctype = ContentType.objects.get_for_model(obj) - + for f in date_fields: - f=f.lstrip('-') + f = f.lstrip('-') date = getattr(obj, f, None) - if isinstance(date, datetime.datetime) or isinstance(date, datetime.date): + if isinstance(date, (datetime.datetime, datetime.date)): break date = None continue - + processed_tags = [] - - # Remove existing items, this ensures tagged items + + # Remove existing items, this ensures tagged items # are updated correctly - SuperTaggedItem.objects.active().filter(content_type=ctype, + SuperTaggedItem.objects.active().filter(content_type=ctype, object_id=obj.pk).delete() if settings.PROCESS_RELATIONS: - SuperTaggedRelationItem.objects.filter(content_type=ctype, + SuperTaggedRelationItem.objects.filter(content_type=ctype, object_id=obj.pk).delete() - + for item in params['fields']: try: d = item.copy() - + field = d.pop('name') proc_type = d.pop('process_type', process_type) comb_fields = d.pop('combine_fields', None) - + if comb_fields is None: data = force_unicode(getattr(obj, field)) else: data = '\n'.join([force_unicode(getattr(obj, item, '')) for item in comb_fields]) - + # Analyze the text (data) result = c.analyze(data) - + entities, relations, topics, socialtags = [], [], [], [] # Process entities, relations and topics if hasattr(result, 'entities'): try: - entities = _processEntities(field, result.entities, + entities = _processEntities(field, result.entities, obj, ctype, proc_type, tags, date) except Exception, e: - if settings.ST_DEBUG: raise Exception("Failed to process Entities: %s" % e) - + if settings.ST_DEBUG: + raise Exception("Failed to process Entities: %s" % e) + if hasattr(result, 'relations') and settings.PROCESS_RELATIONS: try: - relations = _processRelations(field, result.relations, obj, - ctype, proc_type, tags, date) + _processRelations(field, result.relations, obj, ctype, + proc_type, tags, date) except Exception, e: - if settings.ST_DEBUG: raise Exception("Failed to process Relations: %s" % e) + if settings.ST_DEBUG: + raise Exception("Failed to process Relations: %s" % e) if hasattr(result, 'topics') and settings.PROCESS_TOPICS: try: - topics = _processTopics(field, result.topics, obj, + topics = _processTopics(field, result.topics, obj, ctype, tags, date) except Exception, e: - if settings.ST_DEBUG: raise Exception("Failed to process Topics: %s" % e) - + if settings.ST_DEBUG: + raise Exception("Failed to process Topics: %s" % e) + if hasattr(result, 'socialTag') and settings.PROCESS_SOCIALTAGS: try: - socialtags = _processSocialTags(field, result.socialTag, obj, + socialtags = _processSocialTags(field, result.socialTag, obj, ctype, tags, date) except Exception, e: - if settings.ST_DEBUG: raise Exception("Failed to process SocialTags: %s" % e) - + if settings.ST_DEBUG: + raise Exception("Failed to process SocialTags: %s" % e) + processed_tags.extend(entities) processed_tags.extend(topics) processed_tags.extend(socialtags) - + if settings.MARKUP: invalidate_markup_cache(obj, field) - + except Exception, e: - if settings.ST_DEBUG: raise Exception(e) + if settings.ST_DEBUG: + raise Exception(e) continue return processed_tags + def clean_up(obj): """ - When an object is removed, remove all the super tagged items + When an object is removed, remove all the super tagged items and super tagged relation items """ try: cont_type = ContentType.objects.get_for_model(obj) - SuperTaggedItem.objects.filter(content_type=cont_type, + SuperTaggedItem.objects.filter(content_type=cont_type, object_id=obj.pk).delete() - SuperTaggedRelationItem.objects.filter(content_type=cont_type, + SuperTaggedRelationItem.objects.filter(content_type=cont_type, object_id=obj.pk).delete() except Exception, e: - if settings.ST_DEBUG: raise Exception(e) + if settings.ST_DEBUG: + raise Exception(e) # TODO, clean up tags that have no related items? # Same for relations? + def _processEntities(field, data, obj, ctype, process_type, tags, date): """ Process Entities. @@ -223,12 +236,12 @@ def _processEntities(field, data, obj, ctype, process_type, tags, date): entity = e.copy() # Here we convert the given float value to an integer rel = int(float(str(entity.pop('relevance', '0'))) * 1000) - - # Only process tags and items that greater or equal + + # Only process tags and items that greater or equal # to MIN_RELEVANCE setting if rel < settings.MIN_RELEVANCE: continue - + inst = entity.pop('instances', {}) ## Tidy up the encoding for i, j in enumerate(inst): @@ -238,19 +251,18 @@ def _processEntities(field, data, obj, ctype, process_type, tags, date): else: inst[i][k] = v - calais_id = re.match(REF_REGEX, str(entity.pop('__reference'))).group('key') stype = entity.pop('_type', '') # if type is in EXLCUSIONS, continue to next item. if stype.lower() in map(lambda s: s.lower(), settings.EXCLUSIONS): continue - + display_name = entity.pop('name', '') name = display_name.lower() if tags and name not in tags: continue - + slug = slugify(name) tag = None try: @@ -270,22 +282,22 @@ def _processEntities(field, data, obj, ctype, process_type, tags, date): tag = SuperTag.objects.create_alternate(**kwargs) except SuperTag.MultipleObjectsReturned: tag = SuperTag.objects.filter(name__iexact=name)[0] - + tag = tag.substitute or tag - + # If this tag was added to exlcude list, move onto the next item. if not tag.enabled: continue - + tag.properties = entity tag.save() - + # Check to make sure that the entity is not already attached # to the content object, if it is, just append the instances. This # should elimiate entities returned with different names such as # 'Washington' and 'Washington DC' but same id try: - it = SuperTaggedItem.objects.get(tag=tag, content_type=ctype, + it = SuperTaggedItem.objects.get(tag=tag, content_type=ctype, object_id=obj.pk, field=field) it.instances.append(inst) it.item_date = date @@ -295,14 +307,15 @@ def _processEntities(field, data, obj, ctype, process_type, tags, date): it.save() except SuperTaggedItem.DoesNotExist: # Create the record that will associate content to tags - it = SuperTaggedItem.objects.create(tag=tag, - content_type=ctype, object_id=obj.pk, field=field, - process_type=process_type, relevance=rel, instances=inst, + it = SuperTaggedItem.objects.create(tag=tag, + content_type=ctype, object_id=obj.pk, field=field, + process_type=process_type, relevance=rel, instances=inst, item_date=date) processed_tags.append(tag) return processed_tags + def _processRelations(field, data, obj, ctype, process_type, tags, date): """ Process Relations @@ -316,16 +329,16 @@ def _processRelations(field, data, obj, ctype, process_type, tags, date): # If type is in REL_EXLCUSIONS, continue to next item. if rel_type.lower() in map(lambda s: s.lower(), settings.REL_EXLCUSIONS): continue - + props = {} entities = {} # Loop all the items in search of entities (SuperTags). - for k,v in di.items(): + for k, v in di.items(): if isinstance(v, dict): ref = v.pop('__reference', '') else: ref = v - + res = re.match(REF_REGEX, unicode(ref)) if res: entities[k] = res.group('key') @@ -338,9 +351,9 @@ def _processRelations(field, data, obj, ctype, process_type, tags, date): # as the "ID", since the SuperTagRelation needs one "entity" we take one and try to # resolve the other found entities. Entities should already exist from the previous # opertaion "_processEntities" - for entity_key,entity_value in entities.items(): + for entity_key, entity_value in entities.items(): _vals = {} - for entity_temp_key,entity_temp_value in entities_temp.items(): + for entity_temp_key, entity_temp_value in entities_temp.items(): if entity_key != entity_temp_key and entity_value != entity_temp_value: _vals[entity_temp_key] = _getEntityText(entity_temp_value) @@ -348,23 +361,24 @@ def _processRelations(field, data, obj, ctype, process_type, tags, date): if tags and entity_value not in tags: continue - + try: entity = SuperTag.objects.get(calais_id=entity_value) entity = entity.substitute or entity except SuperTag.DoesNotExist: continue - + if not entity.enabled: continue - + rel_item, rel_created = SuperTagRelation.objects.get_or_create( tag=entity, name=entity_key, stype=rel_type, properties=_vals) - SuperTaggedRelationItem.objects.create(relation=rel_item, - content_type=ctype, object_id=obj.pk, field=field, + SuperTaggedRelationItem.objects.create(relation=rel_item, + content_type=ctype, object_id=obj.pk, field=field, process_type=process_type, instances=inst, item_date=date) + def _processTopics(field, data, obj, ctype, tags, date): """ Process Topics, this opertaion is similar to _processEntities, the only @@ -373,16 +387,16 @@ def _processTopics(field, data, obj, ctype, tags, date): processed_tags = [] for di in data: di.pop('__reference') - + calais_id = re.match(REF_REGEX, str(di.pop('category'))).group('key') stype = 'Topic' display_name = di.pop('categoryName', '') name = display_name.lower() - + if tags and name not in tags: continue rel = int(float(str(di.pop('score', '0'))) * 1000) - + slug = slugify(name) tag = None try: @@ -402,21 +416,22 @@ def _processTopics(field, data, obj, ctype, tags, date): tag = SuperTag.objects.create_alternate(**kwargs) except SuperTag.MultipleObjectsReturned: tag = SuperTag.objects.filter(name__iexact=name)[0] - + tag = tag.substitute or tag - + if not tag.enabled: continue tag.properties = di tag.save() - SuperTaggedItem.objects.create(tag=tag, content_type=ctype, + SuperTaggedItem.objects.create(tag=tag, content_type=ctype, object_id=obj.pk, field=field, relevance=rel, item_date=date) processed_tags.append(tag) return processed_tags + def _processSocialTags(field, data, obj, ctype, tags, date): """ Process Topics, this opertaion is similar to _processEntities, the only @@ -426,7 +441,7 @@ def _processSocialTags(field, data, obj, ctype, tags, date): processed_tags = [] for di in data: di.pop('__reference') - + calais_id = re.match(REF_REGEX, str(di.pop('socialTag'))).group('key') stype = 'Social Tag' display_name = di.pop('name', '') @@ -453,21 +468,22 @@ def _processSocialTags(field, data, obj, ctype, tags, date): tag = SuperTag.objects.create_alternate(**kwargs) except SuperTag.MultipleObjectsReturned: tag = SuperTag.objects.filter(name__iexact=name)[0] - + tag = tag.substitute or tag - + if not tag.enabled: continue tag.properties = di tag.save() - SuperTaggedItem.objects.create(tag=tag, content_type=ctype, + SuperTaggedItem.objects.create(tag=tag, content_type=ctype, object_id=obj.pk, field=field, relevance=rel, item_date=date) processed_tags.append(tag) return processed_tags + def _getEntityText(key): """ Try to resolve the entity given the key diff --git a/supertagging/settings.py b/supertagging/settings.py index 1ab97a7..202120b 100644 --- a/supertagging/settings.py +++ b/supertagging/settings.py @@ -2,86 +2,86 @@ import warnings DEFAULT_PROCESSING_DIRECTIVES = { - "contentType": "TEXT/RAW", - "outputFormat": "application/json", - "reltagBaseURL": '', - "calculateRelevanceScore": True, - "enableMetadataType": '', - "docRDFaccessible": True, + "contentType": "TEXT/RAW", + "outputFormat": "application/json", + "reltagBaseURL": '', + "calculateRelevanceScore": True, + "enableMetadataType": '', + "docRDFaccessible": True, } DEFAULT_USER_DIRECTIVES = { - "allowDistribution": False, - "allowSearch": False, + "allowDistribution": False, + "allowSearch": False, "externalID": '', "submitter": "python-calais client v.1.5", } DEFAULT_CALAIS_SETTINGS = { 'API_KEY': '', - # 'USER_DIRECTIVES': DEFAULT_USER_DIRECTIVES, + # 'USER_DIRECTIVES': DEFAULT_USER_DIRECTIVES, # 'PROCESSING_DIRECTIVES': DEFAULT_PROCESSING_DIRECTIVES, 'PROCESS_RELATIONS': False, 'PROCESS_TOPICS': False, 'PROCESS_SOCIALTAGS': False, - 'DEFAULT_PROCESS_TYPE': 'TEXT/RAW', + 'DEFAULT_PROCESS_TYPE': 'TEXT/RAW', } DEFAULT_EXCLUSIONS = { - 'TAG_TYPE_EXCLUSIONS': [], # exclude tags of certian types from saving - 'REL_TYPE_EXCLUSIONS': [], # exclude relations of certian types from saving - 'TAG_TYPE_QUERY_EXCLUSIONS': [], # Tags will be saved, but not returned in - # the queries * NOT IMPLEMENTED * - 'MIN_RELEVANCE': 0, # Minimum relevance to save a tag-object pair (0-1000) + 'TAG_TYPE_EXCLUSIONS': [], # exclude tags of certian types from saving + 'REL_TYPE_EXCLUSIONS': [], # exclude relations of certian types from saving + 'TAG_TYPE_QUERY_EXCLUSIONS': [], # Tags will be saved, but not returned in + # the queries * NOT IMPLEMENTED * + 'MIN_RELEVANCE': 0, # Minimum relevance to save a tag-object pair (0-1000) } DEFAULT_FREEBASE_SETTINGS = { - 'ENABLED': False, # use freebase to disambiguate tags - 'TYPE_MAPPINGS': {}, # maps calais types to freebase types - 'RETRIEVE_DESCRIPTIONS': False, # True: attempt to retreive description - # for the tag via Freebase and save it to - # the description field + 'ENABLED': False, # use freebase to disambiguate tags + 'TYPE_MAPPINGS': {}, # maps calais types to freebase types + 'RETRIEVE_DESCRIPTIONS': False, # True: attempt to retreive description + # for the tag via Freebase and save it to + # the description field # The first part of the url to retreive descriptions from freebase 'DESCRIPTION_URL': "http://www.freebase.com/api/trans/raw", } DEFAULT_MARKUP_SETTINGS = { - 'ENABLED': False, # True: Automatically provide a version of the content - # with the tags marked with links. - 'FIELD_SUFFIX': "tagged", # The suffix of the field created when using markup. - 'EXCLUDE': [], # List of strings that will be excluded from being marked up, eg: his, her, him etc. - 'CONTENT_CACHE_TIMEOUT': 3600, # Integer for the cache timeout for the markup content. - 'MIN_RELEVANCE': 0, # Minimum relevance of a tag to include it in - # automatic markup of the content (0-1000) + 'ENABLED': False, # True: Automatically provide a version of the content + # with the tags marked with links. + 'FIELD_SUFFIX': "tagged", # The suffix of the field created when using markup. + 'EXCLUDE': [], # List of strings that will be excluded from being marked up, eg: his, her, him etc. + 'CONTENT_CACHE_TIMEOUT': 3600, # Integer for the cache timeout for the markup content. + 'MIN_RELEVANCE': 0, # Minimum relevance of a tag to include it in + # automatic markup of the content (0-1000) } DEFAULT_SETTINGS = { - 'ENABLED': False, # Enable supertagging. This will allow starting and - # stopping tag processing while preserving AUTO_PROCESS - 'DEBUG': False, # If True, raise errors when errors occur - 'WATCHED_FIELDS': {}, # The models/fields to process - 'INCLUDE_DISPLAY_FIELDS': True, # True: include fields: display_name, description, icon, related - 'AUTO_PROCESS': False, # Set up post save and delete signals - 'ONLY_NON_TAGGED_OBJECTS': False, # If the save signal is used, - # True: tell the handler to only add objects - # that have never had tags. Objects - # that have tags but need re-processing - # must be added to the queue manually. - # False: handle all objects - 'RESOLVE_PROPERTY_KEYS': True, # True: resolve related tags' name, - # False: keep the UID - 'REGISTER_MODELS': False, # True: add an attribute to your model(s) to - # retrieve the tags. - 'SUBSTITUTE_TAG_UPDATE': False, # If True, when a substitute is supplied all the Tagged Items and Relation + 'ENABLED': False, # Enable supertagging. This will allow starting and + # stopping tag processing while preserving AUTO_PROCESS + 'DEBUG': False, # If True, raise errors when errors occur + 'WATCHED_FIELDS': {}, # The models/fields to process + 'INCLUDE_DISPLAY_FIELDS': True, # True: include fields: display_name, description, icon, related + 'AUTO_PROCESS': False, # Set up post save and delete signals + 'ONLY_NON_TAGGED_OBJECTS': False, # If the save signal is used, + # True: tell the handler to only add objects + # that have never had tags. Objects + # that have tags but need re-processing + # must be added to the queue manually. + # False: handle all objects + 'RESOLVE_PROPERTY_KEYS': True, # True: resolve related tags' name, + # False: keep the UID + 'REGISTER_MODELS': False, # True: add an attribute to your model(s) to + # retrieve the tags. + 'SUBSTITUTE_TAG_UPDATE': False, # If True, when a substitute is supplied all the Tagged Items and Relation # Tagged Items will be set with the substitute tag. If False, the Tagged Items # and the Relation Tagged Items will still have the original tag. This is a # way to preserve the old tag data. - 'REMOVE_REL_ON_DISABLE': False, # True: all related content to a tag is - # removed (items from models - # `SuperTaggedItem` and `SuperTaggedRelationItem`) - 'FILE_STORAGE': settings.DEFAULT_FILE_STORAGE, # For the tag icon - 'USE_QUEUE': False, # True: add objects to a queue for later processing - # False: process the item on save. - 'CONTENTTYPE_NAME_MAPPING': {}, # Names used enstead of integers when displaying the content. - # EX: {'stories': 322, 'photos': 129, 'entries': 102, 'polls': 754} - # Where the value is the actual content type id and the key is the name - # used in the url. - # supertagging/tags/barack_obama/stories/ - # supertagging/tags/world_cup/photos/ + 'REMOVE_REL_ON_DISABLE': False, # True: all related content to a tag is + # removed (items from models + # `SuperTaggedItem` and `SuperTaggedRelationItem`) + 'FILE_STORAGE': settings.DEFAULT_FILE_STORAGE, # For the tag icon + 'USE_QUEUE': False, # True: add objects to a queue for later processing + # False: process the item on save. + 'CONTENTTYPE_NAME_MAPPING': {}, # Names used enstead of integers when displaying the content. + # EX: {'stories': 322, 'photos': 129, 'entries': 102, 'polls': 754} + # Where the value is the actual content type id and the key is the name + # used in the url. + # supertagging/tags/barack_obama/stories/ + # supertagging/tags/world_cup/photos/ # 'OPEN_CALAIS': DEFAULT_CALAIS_SETTINGS, # 'EXCLUSIONS': DEFAULT_EXCLUSIONS, # 'MARKUP': DEFAULT_MARKUP_SETTINGS, @@ -121,7 +121,7 @@ warnings.warn(ERR_MSG % (dep_setting, new_setting), DeprecationWarning) USER_SETTINGS[new_setting] = getattr(settings, dep_setting) globals().update({short_name: USER_SETTINGS[new_setting]}) - + DEP_CALAIS = ( ('SUPERTAGGING_CALAIS_USER_DIRECTIVES', 'USER_DIRECTIVES', 'USER_DIR'), diff --git a/supertagging/tests.py b/supertagging/tests.py index 3fa7514..b2e6494 100644 --- a/supertagging/tests.py +++ b/supertagging/tests.py @@ -5,23 +5,26 @@ from django.db import models from supertagging.fields import PickledObjectField + class TestingModel(models.Model): pickle_field = PickledObjectField() + class TestCustomDataType(str): pass + class PickledObjectFieldTests(TestCase): def setUp(self): self.testing_data = ( - {1:1, 2:4, 3:6, 4:8, 5:10}, + {1: 1, 2: 4, 3: 6, 4: 8, 5: 10}, 'Hello World', (1, 2, 3, 4, 5), [1, 2, 3, 4, 5], TestCustomDataType('Hello World'), ) return super(PickledObjectFieldTests, self).setUp() - + def testDataIntegriry(self): """Tests that data remains the same when saved to and fetched from the database.""" for value in self.testing_data: @@ -30,7 +33,7 @@ def testDataIntegriry(self): model_test = TestingModel.objects.get(id__exact=model_test.id) self.assertEquals(value, model_test.pickle_field) model_test.delete() - + def testLookups(self): """Tests that lookups can be performed on data once stored in the database.""" for value in self.testing_data: @@ -38,4 +41,3 @@ def testLookups(self): model_test.save() self.assertEquals(value, TestingModel.objects.get(pickle_field__exact=value).pickle_field) model_test.delete() - \ No newline at end of file diff --git a/supertagging/utils.py b/supertagging/utils.py index a37979d..88891eb 100644 --- a/supertagging/utils.py +++ b/supertagging/utils.py @@ -7,22 +7,16 @@ from django.db.models.query import QuerySet from django.utils.encoding import force_unicode from django.utils.translation import ugettext as _ -from django.template.defaultfilters import slugify -from django.template.loader import render_to_string, get_template +from django.template.loader import render_to_string, select_template from supertagging import settings -# Python 2.3 compatibility -try: - set -except NameError: - from sets import Set as set -from operator import itemgetter def tag_instance_cmp(x, y): if isinstance(x, dict) and isinstance(y, dict): - return cmp(x['offset'],y['offset']) + return cmp(x['offset'], y['offset']) return cmp(1, 1) + def parse_tag_input(input): """ Parses tag input, with multiple word input being activated and @@ -92,7 +86,8 @@ def parse_tag_input(input): words = list(set(words)) words.sort() return words - + + def split_strip(input, delimiter=u','): """ Splits ``input`` on ``delimiter``, stripping each resulting string @@ -104,6 +99,7 @@ def split_strip(input, delimiter=u','): words = [w.strip() for w in input.split(delimiter)] return [w for w in words if w] + def edit_string_for_tags(tags): """ Given list of ``SuperTag`` instances, creates a string representation of @@ -134,6 +130,7 @@ def edit_string_for_tags(tags): glue = u' ' return glue.join(names) + def get_queryset_and_model(queryset_or_model): """ Given a ``QuerySet`` or a ``Model``, returns a two-tuple of @@ -147,6 +144,7 @@ def get_queryset_and_model(queryset_or_model): except AttributeError: return queryset_or_model._default_manager.all(), queryset_or_model + def get_tag_list(tags): """ Utility function for accepting tag input in a flexible manner. @@ -165,7 +163,6 @@ def get_tag_list(tags): * A list or tuple of ``SuperTag`` objects. * A ``SuperTag`` ``QuerySet``. - """ from supertagging.models import SuperTag if isinstance(tags, SuperTag): @@ -173,8 +170,8 @@ def get_tag_list(tags): elif isinstance(tags, QuerySet) and tags.model is SuperTag: return tags elif isinstance(tags, types.StringTypes): - return SuperTag.objects.filter(name__in=parse_tag_input(tags))\ - |SuperTag.objects.filter(slug__in=parse_tag_input(tags)) + return (SuperTag.objects.filter(name__in=parse_tag_input(tags)) + | SuperTag.objects.filter(slug__in=parse_tag_input(tags))) elif isinstance(tags, (types.ListType, types.TupleType)): if len(tags) == 0: return tags @@ -182,23 +179,24 @@ def get_tag_list(tags): for item in tags: if isinstance(item, types.StringTypes): contents.add('string') - elif isinstance(item, Tag): - contents.add('tag') elif isinstance(item, (types.IntType, types.LongType)): contents.add('int') if len(contents) == 1: if 'string' in contents: - return SuperTag.objects.filter(name__in=[force_unicode(tag) for tag in tags])\ - |SuperTag.objects.filter(slug__in=[force_unicode(tag) for tag in tags]) + tagstrings = [force_unicode(tag) for tag in tags] + return (SuperTag.objects.filter(name__in=tagstrings) + | SuperTag.objects.filter(slug__in=tagstrings)) elif 'tag' in contents: return tags elif 'int' in contents: return SuperTag.objects.filter(id__in=tags) else: - raise ValueError(_('If a list or tuple of tags is provided, they must all be tag names, SuperTag objects or Tag ids.')) + raise ValueError(_('If a list or tuple of tags is provided, they ' + 'must all be tag names, SuperTag objects or Tag ids.')) else: raise ValueError(_('The tag input given was invalid.')) + def get_tag(tag): """ Utility function for accepting single tag input in a flexible @@ -227,10 +225,12 @@ def get_tag(tag): # Font size distribution algorithms LOGARITHMIC, LINEAR = 1, 2 + def _calculate_thresholds(min_weight, max_weight, steps): delta = (max_weight - min_weight) / float(steps) return [min_weight + i * delta for i in range(1, steps + 1)] + def _calculate_tag_weight(weight, max_weight, distribution): """ Logarithmic tag weight calculation is based on code from the @@ -244,6 +244,7 @@ def _calculate_tag_weight(weight, max_weight, distribution): return math.log(weight) * max_weight / math.log(max_weight) raise ValueError(_('Invalid distribution algorithm specified: %s.') % distribution) + def calculate_cloud(tags, steps=4, distribution=LOGARITHMIC): """ Add a ``font_size`` attribute to each tag according to the @@ -270,7 +271,7 @@ def calculate_cloud(tags, steps=4, distribution=LOGARITHMIC): tag.font_size = i + 1 font_set = True return tags - + ########################### # Freebase Util Functions # ########################### @@ -283,6 +284,7 @@ def calculate_cloud(tags, steps=4, distribution=LOGARITHMIC): # The key from freebase that will have the topic description FREEBASE_DESC_KEY = "/common/topic/article" + def fix_name_for_freebase(value): """ Takes a name and replaces spaces with underscores, removes periods @@ -293,40 +295,47 @@ def fix_name_for_freebase(value): word = word.replace(".", "") words.append(word.title()) return "_".join(words) - + + def retrieve_freebase_name(name, stype): if not freebase: return name - + search_key = fix_name_for_freebase(name) fb_type = settings.FREEBASE_TYPE_MAPPINGS.get(stype, None) value = None try: # Try to get the exact match - value = freebase.mqlread( - {"name": None, "type":fb_type or [], - "key": {"value": search_key}}) + value = freebase.mqlread({ + "name": None, + "type": fb_type or [], + "key": {"value": search_key} + }) except: try: # Try to get a results has a generator and return its top result - values = freebase.mqlreaditer( - {"name": None, "type":fb_type or [], - "key": {"value": search_key}}) + values = freebase.mqlreaditer({ + "name": None, + "type": fb_type or [], + "key": {"value": search_key} + }) value = values.next() except Exception, e: # Only print error as freebase is only optional - if settings.ST_DEBUG: print "Error using `freebase`: %s" % e - + if settings.ST_DEBUG: + print "Error using `freebase`: %s" % e + if value: return value["name"] return name - + + def retrieve_freebase_desc(name, stype): if not freebase: return "" - + print "Retrieving the description for %s" % name - + fb_type = settings.FREEBASE_TYPE_MAPPINGS.get(stype, None) value, data = None, "" try: @@ -341,8 +350,9 @@ def retrieve_freebase_desc(name, stype): value = values.next() except Exception, e: # Only print error as freebase is only optional - if settings.ST_DEBUG: print "Error using `freebase`: %s" % e - + if settings.ST_DEBUG: + print "Error using `freebase`: %s" % e + if value and FREEBASE_DESC_KEY in value and value[FREEBASE_DESC_KEY]: guid = value[FREEBASE_DESC_KEY][0].get("id", None) if not guid: @@ -351,173 +361,47 @@ def retrieve_freebase_desc(name, stype): import urllib desc_url = "%s%s" % (settings.FREEBASE_DESCRIPTION_URL, guid) sock = urllib.urlopen(desc_url) - data = sock.read() + data = sock.read() sock.close() except Exception, e: - if settings.ST_DEBUG: print "Error getting description from freebase for tag \"%s\" - %s" % (name, e) - + if settings.ST_DEBUG: + print "Error getting description from freebase for tag \"%s\" - %s" % (name, e) + return data - + + ################ # Render Utils # ################ - def render_item(item, stype, template, suffix, template_path='supertagging/render', context={}): """ Use to render tags, relations, tagged items and tagger relations. """ t, model, app, = None, "", "" - + if item: model = item.content_type.model.lower() app = item.content_type.app_label.lower() - - tp = "%s/%s" % (template_path, (stype or "")) - - try: - # Retreive the template passed in - t = get_template(template) - except: - if suffix: - try: - # Retrieve the template based off of type and the content object with a suffix - t = get_template('%s/%s__%s__%s.html' % ( - tp, app, model, suffix.lower())) - except: - pass - else: - try: - # Retrieve the template based off of type and the content object - t = get_template('%s/%s__%s.html' % ( - tp, app, model)) - except: - pass - if not t: - if suffix: - try: - # Retrieve the template without the app/model with suffix - t = get_template('%s/default__%s.html' % (tp, suffix)) - except: - pass - else: - try: - # Retrieve the template without the app/model - t = get_template('%s/default.html' % tp) - except: - try: - # Retreive the default template using just the starting template path - t = get_template('%s/default.html' % template_path) - except: - pass - - if not t: return None - - # Render the template - ret = render_to_string(t.name, context) - - return ret - -""" -Provides compatibility with Django 1.1 - -Copied from django.contrib.admin.util -""" -from django.db import models -from django.utils.encoding import force_unicode, smart_unicode, smart_str + tp = "%s/%s" % (template_path, (stype or "")) -def lookup_field(name, obj, model_admin=None): - opts = obj._meta - try: - f = opts.get_field(name) - except models.FieldDoesNotExist: - # For non-field values, the value is either a method, property or - # returned via a callable. - if callable(name): - attr = name - value = attr(obj) - elif (model_admin is not None and hasattr(model_admin, name) and - not name == '__str__' and not name == '__unicode__'): - attr = getattr(model_admin, name) - value = attr(obj) - else: - attr = getattr(obj, name) - if callable(attr): - value = attr() - else: - value = attr - f = None - else: - attr = None - value = getattr(obj, name) - return f, attr, value + # Retreive the template passed in + # Retrieve the template based off of type and the content object with a suffix + # Retrieve the template based off of type and the content object + # Retrieve the template without the app/model with suffix + # Retrieve the template without the app/model + # Retreive the default template using just the starting template path + templates = ( + template, + '%s/%s__%s__%s.html' % (tp, app, model, suffix.lower()), + '%s/%s__%s.html' % (tp, app, model), + '%s/default__%s.html' % (tp, suffix.lower()), + '%s/default.html' % tp, + '%s/default.html' % template_path, + ) + + t = select_template(templates) + ret = render_to_string(t.name, context) -def label_for_field(name, model, model_admin=None, return_attr=False): - """ - Returns a sensible label for a field name. The name can be a callable or the - name of an object attributes, as well as a genuine fields. If return_attr is - True, the resolved attribute (which could be a callable) is also returned. - This will be None if (and only if) the name refers to a field. - """ - attr = None - try: - field = model._meta.get_field_by_name(name)[0] - if isinstance(field, RelatedObject): - label = field.opts.verbose_name - else: - label = field.verbose_name - except models.FieldDoesNotExist: - if name == "__unicode__": - label = force_unicode(model._meta.verbose_name) - attr = unicode - elif name == "__str__": - label = smart_str(model._meta.verbose_name) - attr = str - else: - if callable(name): - attr = name - elif model_admin is not None and hasattr(model_admin, name): - attr = getattr(model_admin, name) - elif hasattr(model, name): - attr = getattr(model, name) - else: - message = "Unable to lookup '%s' on %s" % (name, model._meta.object_name) - if model_admin: - message += " or %s" % (model_admin.__class__.__name__,) - raise AttributeError(message) - - if hasattr(attr, "short_description"): - label = attr.short_description - elif callable(attr): - if attr.__name__ == "": - label = "--" - else: - label = pretty_name(attr.__name__) - else: - label = pretty_name(name) - if return_attr: - return (label, attr) - else: - return label - -def display_for_field(value, field): - from django.contrib.admin.templatetags.admin_list import _boolean_icon - from django.contrib.admin.views.main import EMPTY_CHANGELIST_VALUE - - if field.flatchoices: - return dict(field.flatchoices).get(value, EMPTY_CHANGELIST_VALUE) - # NullBooleanField needs special-case null-handling, so it comes - # before the general null test. - elif isinstance(field, models.BooleanField) or isinstance(field, models.NullBooleanField): - return _boolean_icon(value) - elif value is None: - return EMPTY_CHANGELIST_VALUE - elif isinstance(field, models.DateField) or isinstance(field, models.TimeField): - return formats.localize(value) - elif isinstance(field, models.DecimalField): - return formats.number_format(value, field.decimal_places) - elif isinstance(field, models.FloatField): - return formats.number_format(value) - else: - return smart_unicode(value) + return ret From 63373075e4e363ce0feb7391ce0a61c4b466ddd5 Mon Sep 17 00:00:00 2001 From: Corey Oordt Date: Sun, 24 Mar 2013 12:07:48 -0500 Subject: [PATCH 2/9] Forgot a file for PEP 8 cleaning --- .../templatetags/supertagging_tags.py | 178 +++++++++--------- 1 file changed, 93 insertions(+), 85 deletions(-) diff --git a/supertagging/templatetags/supertagging_tags.py b/supertagging/templatetags/supertagging_tags.py index 1e1913f..89bbfe9 100644 --- a/supertagging/templatetags/supertagging_tags.py +++ b/supertagging/templatetags/supertagging_tags.py @@ -1,30 +1,25 @@ # Most template tags were borrowed from django-tagging. -import django -from django.db.models import get_model -from django.template import (Library, Node, TemplateSyntaxError, Variable, - resolve_variable, VariableDoesNotExist) -from django.utils.translation import ugettext as _ +from django.contrib.admin.templatetags.admin_list import _boolean_icon, DOT +from django.contrib.admin.util import lookup_field, display_for_field from django.contrib.admin.views.main import (ALL_VAR, EMPTY_CHANGELIST_VALUE, - ORDER_VAR, ORDER_TYPE_VAR, PAGE_VAR, SEARCH_VAR) + ORDER_VAR) from django.core.exceptions import ObjectDoesNotExist -try: - from django.contrib.admin.util import (lookup_field, display_for_field, - label_for_field) -except ImportError: - from supertagging.utils import (lookup_field, display_for_field, - label_for_field) -from django.utils.safestring import mark_safe +from django.db import models +from django.db.models import get_model +from django.template import Library, Node, TemplateSyntaxError, Variable from django.utils.encoding import smart_unicode, force_unicode from django.utils.html import escape, conditional_escape -from django.db import models +from django.utils.safestring import mark_safe +from django.utils.translation import ugettext as _ -from supertagging.models import (SuperTag, SuperTaggedItem, SuperTagRelation, +from supertagging.models import (SuperTag, SuperTaggedItem, SuperTagRelation, SuperTaggedRelationItem) from supertagging.utils import LINEAR, LOGARITHMIC register = Library() + class TagsForModelNode(Node): def __init__(self, model, context_var, counts, **kwargs): self.model = model @@ -36,9 +31,9 @@ def render(self, context): model = get_model(*self.model.split('.')) if model is None: raise TemplateSyntaxError(_('supertags_for_model tag was given an invalid model: %s') % self.model) - + if 'filters' in self.kwargs and isinstance(self.kwargs['filters'], dict): - for k,v in self.kwargs['filters'].items(): + for k, v in self.kwargs['filters'].items(): try: v = Variable(v).resolve(context) self.kwargs['filters'][k] = v @@ -47,6 +42,7 @@ def render(self, context): context[self.context_var] = SuperTag.objects.usage_for_model(model, counts=self.counts) return '' + class TagCloudForModelNode(Node): def __init__(self, model, context_var, **kwargs): self.model = model @@ -57,9 +53,9 @@ def render(self, context): model = get_model(*self.model.split('.')) if model is None: raise TemplateSyntaxError(_('supertag_cloud_for_model tag was given an invalid model: %s') % self.model) - + if 'filters' in self.kwargs and isinstance(self.kwargs['filters'], dict): - for k,v in self.kwargs['filters'].items(): + for k, v in self.kwargs['filters'].items(): try: v = Variable(v).resolve(context) self.kwargs['filters'][k] = v @@ -69,6 +65,7 @@ def render(self, context): SuperTag.objects.cloud_for_model(model, **self.kwargs) return '' + class TagsForObjectNode(Node): def __init__(self, obj, context_var, **kwargs): self.obj = Variable(obj) @@ -80,6 +77,7 @@ def render(self, context): SuperTag.objects.get_for_object(self.obj.resolve(context), **self.kwargs) return '' + class TaggedObjectsNode(Node): def __init__(self, tag, model, context_var): self.tag = Variable(tag) @@ -94,6 +92,7 @@ def render(self, context): SuperTaggedItem.objects.get_by_model(model, self.tag.resolve(context)) return '' + class RelatedObjectsForObjectNode(Node): def __init__(self, obj, model, context_var, **kwargs): self.obj = Variable(obj) @@ -109,6 +108,7 @@ def render(self, context): SuperTaggedItem.objects.get_related(self.obj.resolve(context), model, **self.kwargs) return '' + def do_tags_for_model(parser, token): """ Retrieves a list of ``Tag`` objects associated with a given model @@ -132,13 +132,14 @@ def do_tags_for_model(parser, token): {% supertags_for_model products.Widget as widget_tags %} {% supertags_for_model products.Widget as widget_tags with counts %} - + {% supertags_for_model products.Widget as widget_tags with counts product__category__pk=1 %} {% supertags_for_model products.Widget as widget_tags with counts product__category__pk=category.pk %} """ bits = token.contents.split() len_bits = len(bits) + kwargs = {} if not len_bits > 3: raise TemplateSyntaxError(_('%s tag requires more than two arguments') % bits[0]) if bits[2] != 'as': @@ -159,7 +160,7 @@ def do_tags_for_model(parser, token): else: # The remaining bits should be consider extra query params kwargs['filters'][name] = value - + except ValueError: raise TemplateSyntaxError(_("%(tag)s tag was given a badly formatted option: '%(option)s'") % { 'tag': bits[0], @@ -170,9 +171,10 @@ def do_tags_for_model(parser, token): if bits[5] != 'counts': raise TemplateSyntaxError(_("if given, fifth argument to %s tag must be 'counts'") % bits[0]) if len_bits == 4: - return TagsForModelNode(bits[1], bits[3], counts=False) + return TagsForModelNode(bits[1], bits[3], counts=False, **kwargs) else: - return TagsForModelNode(bits[1], bits[3], counts=True) + return TagsForModelNode(bits[1], bits[3], counts=True, **kwargs) + def do_tag_cloud_for_model(parser, token): """ @@ -203,28 +205,31 @@ def do_tag_cloud_for_model(parser, token): ``distribution`` One of ``linear`` or ``log``. Defines the font-size distribution algorithm to use when generating the tag cloud. - - If anything else is speficied in the options, it will be consider + + If anything else is speficied in the options, it will be consider extra filter params Examples:: {% supertag_cloud_for_model products.Widget as widget_tags %} {% supertag_cloud_for_model products.Widget as widget_tags with steps=9 min_count=3 distribution=log %} - + {% supertag_cloud_for_model products.Widget as widget_tags with steps=9 min_count=3 distrubution=log product__category__pk=1 %} {% supertag_cloud_for_model products.Widget as widget_tags with steps=9 min_count=3 distrubution=log product__category__pk=category.pk %} """ bits = token.contents.split() len_bits = len(bits) if not len_bits > 3: - raise TemplateSyntaxError(_('%s tag requires more than two arguments') % bits[0]) + raise TemplateSyntaxError( + _('%s tag requires more than two arguments') % bits[0]) if bits[2] != 'as': - raise TemplateSyntaxError(_("second argument to %s tag must be 'as'") % bits[0]) + raise TemplateSyntaxError( + _("second argument to %s tag must be 'as'") % bits[0]) kwargs = {'filters': {}} if len_bits > 5: if bits[4] != 'with': - raise TemplateSyntaxError(_("if given, fourth argument to %s tag must be 'with'") % bits[0]) + raise TemplateSyntaxError( + _("if given, fourth argument to %s tag must be 'with'") % bits[0]) for i in range(5, len_bits): try: name, value = bits[i].split('=') @@ -249,7 +254,7 @@ def do_tag_cloud_for_model(parser, token): else: # The remaining bits should be consider extra query params kwargs['filters'][name] = value - + except ValueError: raise TemplateSyntaxError(_("%(tag)s tag was given a badly formatted option: '%(option)s'") % { 'tag': bits[0], @@ -257,6 +262,7 @@ def do_tag_cloud_for_model(parser, token): }) return TagCloudForModelNode(bits[1], bits[3], **kwargs) + def do_tags_for_object(parser, token): """ Retrieves a list of ``Tag`` objects associated with an object and @@ -269,9 +275,9 @@ def do_tags_for_object(parser, token): Example:: {% supertags_for_object foo_object as tag_list %} - + {% supertags_for_object foo_object as tag_list with field=story %} - + """ bits = token.contents.split() len_bits = len(bits) @@ -302,6 +308,7 @@ def do_tags_for_object(parser, token): }) return TagsForObjectNode(bits[1], bits[3], **kwargs) + def do_tagged_objects(parser, token): """ Retrieves a list of instances of a given model which are tagged with @@ -329,6 +336,7 @@ def do_tagged_objects(parser, token): raise TemplateSyntaxError(_("fourth argument to %s tag must be 'as'") % bits[0]) return TaggedObjectsNode(bits[1], bits[3], bits[5]) + def do_related_objects_for_object(parser, token): """ Retrieves a list of related objects of a given model which shares tags @@ -345,7 +353,7 @@ def do_related_objects_for_object(parser, token): Example:: {% related_objects_for_object show in tv.Show as related_objects %} - + {% related_objects_for_object show in tv.Show as related_objects with min_relevance=500 %} """ @@ -406,8 +414,8 @@ def render(self, context): SuperTagRelation.objects.get_for_tag( self.obj.resolve(context), **kwargs) return '' - - + + class RelationsForObjectNode(Node): def __init__(self, obj, context_var, stype=None): self.obj = Variable(obj) @@ -422,8 +430,8 @@ def render(self, context): SuperTaggedRelationItem.objects.get_for_object( self.obj.resolve(context), **kwargs) return '' - - + + class RelationsForTagInObjectNode(Node): def __init__(self, tag, obj, context_var): self.obj = Variable(obj) @@ -459,61 +467,62 @@ def do_relations_for_tag(parser, token): raise TemplateSyntaxError(_('%s tag requires at least three arguments and at most six arguments') % bits[0]) if bits[2] != 'as': raise TemplateSyntaxError(_("Second argument for %s tag must be 'as'") % bits[0]) - + if len(bits) > 4: if bits[4] != "with": - raise TemplateSyntacError(_("Fouth argument for %s tag must be 'with'") % bits[0]) - + raise TemplateSyntaxError(_("Fouth argument for %s tag must be 'with'") % bits[0]) + if len(bits[5].split("=")) == 2 and bits[5].split("=")[0] == 'type': - return RelationsForTagNode(bits[1], bits[3], bits[5].split("=")[1]) + return RelationsForTagNode(bits[1], bits[3], bits[5].split("=")[1]) else: - raise TemplateSyntacError(_("Last argument for %s tag must be in the format type=[TYPE]") % bits[0]) + raise TemplateSyntaxError(_("Last argument for %s tag must be in the format type=[TYPE]") % bits[0]) return RelationsForTagNode(bits[1], bits[3]) - + def do_relations_for_object(parser, token): """ Retrieves a list of ``Relations`` for a given object - + Useage:: - + {% relations_for_object [object] as [varname] %} {% relations_for_object [object] as [varname] with [type=TYPE]} - + Example:: - + {% relations_for_object story as story_relations %} {% relations_for_object story as story_relations with type=Quotation %} - + """ bits = token.contents.split() if len(bits) < 4 or len(bits) > 6: raise TemplateSyntaxError(_('%s tag requires at least three arguments and at most six arguments') % bits[0]) if bits[2] != 'as': raise TemplateSyntaxError(_("Second argument for %s tag must be 'as'") % bits[0]) - + if len(bits) > 4: if bits[4] != "with": - raise TemplateSyntacError(_("Fouth argument for %s tag must be 'with'") % bits[0]) - + raise TemplateSyntaxError(_("Fouth argument for %s tag must be 'with'") % bits[0]) + if len(bits[5].split("=")) == 2 and bits[5].split("=")[0] == 'type': - return RelationsForObjectNode(bits[1], bits[3], bits[5].split("=")[1]) + return RelationsForObjectNode(bits[1], bits[3], bits[5].split("=")[1]) else: - raise TemplateSyntacError(_("Last argument for %s tag must be in the format type=[TYPE]") % bits[0]) + raise TemplateSyntaxError(_("Last argument for %s tag must be in the format type=[TYPE]") % bits[0]) return RelationsForObjectNode(bits[1], bits[3]) - + + def do_relations_for_tag_in_object(parser, token): """ Retrieves a list of ``Relations`` for a given tag that is also in object - + Useage:: - + {% relations_for [tag] in [object] as [varname] %} - + Example:: - + {% relations_for state_tag in obj as obj_relations %} - + """ bits = token.contents.split() if len(bits) != 6: @@ -523,10 +532,7 @@ def do_relations_for_tag_in_object(parser, token): if bits[4] != 'as': raise TemplateSyntaxError(_("Second argument to %s tag must be 'as'") % bits[0]) return RelationsForTagInObjectNode(bits[1], bits[3], bits[5]) - - - register.tag('relations_for_supertag', do_relations_for_tag) register.tag('relations_for_object', do_relations_for_object) register.tag('relations_for', do_relations_for_tag_in_object) @@ -537,7 +543,7 @@ def __init__(self, obj, template=None, suffix=None): self.obj = obj self.template = template self.suffix = suffix - + def render(self, context): suffix, template = self.suffix, self.template try: @@ -547,75 +553,77 @@ def render(self, context): if isinstance(obj, c): isinst = True break - + if not isinst: return None except: return None - + return obj.render(template=template, suffix=suffix) - - + + def do_render_item(parser, token): """ {% supertag_render [SuperTag or SuperTaggedItem or SuperTagRelation or SuperTaggedRelationItem] [with] [suffix=S] [template=T] %} {% supertag_render tag %} {% supertag_render tagged_item with suffix=custom %} {% supertag_render rel_item with template=mycustomtemplates/supertags/custom.html %} - + Only suffix OR template can be specified, but not both. """ argv = token.contents.split() argc = len(argv) - + if argc < 2 or argc > 4: - raise TemplateSyntaxError, "Tag %s takes either two or four arguments." % argv[0] - + raise TemplateSyntaxError("Tag %s takes either two or four arguments." % argv[0]) + if argc == 2: return RenderItemNode(argv[1]) else: if argv[2] != 'with': - raise TemplateSyntaxError, 'Second argument must be "with" for tag %s.' % argv[0] + raise TemplateSyntaxError('Second argument must be "with" for tag %s.' % argv[0]) extra = argv[3].split('=') if len(extra) != 2: - raise TemplateSyntaxError, "Last argument must be formated correctly for tag %s." % argv[0] + raise TemplateSyntaxError("Last argument must be formated correctly for tag %s." % argv[0]) if not extra[0] in ['suffix', 'template']: - raise TemplateSyntaxError, "Last argment must of either suffix or template for tag %s." % argv[0] - - kwargs = {str(extra[0]): extra[1],} + raise TemplateSyntaxError("Last argment must of either suffix or template for tag %s." % argv[0]) + + kwargs = {str(extra[0]): extra[1], } return RenderItemNode(argv[1], **kwargs) - + register.tag('supertag_render', do_render_item) class IsSupertaggableNode(Node): def __init__(self, obj): self.obj = obj - + def render(self, context): from django.contrib.contenttypes.models import ContentType from supertagging.settings import WATCHED_FIELDS - + obj = Variable(self.obj).resolve(context) ctype = ContentType.objects.get_for_model(obj) result = "%s.%s" % (ctype.app_label, ctype.model) in WATCHED_FIELDS context['is_supertaggable'] = result return '' + def do_is_supertaggable(parser, token): """ {% is_supertaggable obj %} - + Sets context variable "is_supertaggable" to True or False """ try: tag_name, obj = token.split_contents() except ValueError: - raise template.TemplateSyntaxError("%r tag requires exactly one argument" % token.contents.split()[0]) + raise TemplateSyntaxError("%r tag requires exactly one argument" % token.contents.split()[0]) return IsSupertaggableNode(obj) register.tag('is_supertaggable', do_is_supertaggable) + def items_for_result(cl, result, form): """ Generates the actual list of data. @@ -662,9 +670,7 @@ def items_for_result(cl, result, form): result_repr = mark_safe(' ') # If list_display_links not defined, add the link tag to the first field if (first and not cl.list_display_links) or field_name in cl.list_display_links: - table_tag = {True:'th', False:'td'}[first] - if form and django.VERSION[1] == 1: # Django 1.1 - result_repr += force_unicode(form[cl.model._meta.pk.name]) + table_tag = {True: 'th', False: 'td'}[first] first = False # Convert the pk to something that can be used in Javascript. # Problem cases are long ints (23L) and non-ASCII strings. @@ -709,6 +715,7 @@ def results(cl): for res in cl.result_list: yield list(items_for_result(cl, res, None)) + @register.inclusion_tag("admin/supertagging/supertaggeditem/change_list_results.html") def supertaggeditem_result_list(cl): """ @@ -719,7 +726,7 @@ def supertaggeditem_result_list(cl): from django.contrib.admin.templatetags.admin_list import result_hidden_fields except ImportError: result_hidden_fields = lambda x: [] - + headers = list(result_headers(cl)) cl.list_editable = cl.model_admin.list_editable return {'cl': cl, @@ -728,6 +735,7 @@ def supertaggeditem_result_list(cl): 'reset_sorting_url': cl.get_query_string(remove=[ORDER_VAR]), 'results': list(results(cl))} + @register.inclusion_tag("admin/supertagging/supertaggeditem/pagination.html") def supertaggeditem_pagination(cl): """ From ce485cd47b443ed42375c1dd656389589c475d1e Mon Sep 17 00:00:00 2001 From: Corey Oordt Date: Sun, 24 Mar 2013 19:16:50 -0500 Subject: [PATCH 3/9] Added a fix for Django 1.5 (possibly 1.4) may break backwards compatibility. --- supertagging/admin.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/supertagging/admin.py b/supertagging/admin.py index 104a1b2..949ce32 100644 --- a/supertagging/admin.py +++ b/supertagging/admin.py @@ -16,7 +16,9 @@ class SupertagChangeList(ChangeList): """ Lets list_editable work even if it is a popup """ - def __init__(self, request, model, list_display, list_display_links, list_filter, date_hierarchy, search_fields, list_select_related, list_per_page, list_editable, model_admin): + def __init__(self, request, model, list_display, list_display_links, + list_filter, date_hierarchy, search_fields, list_select_related, + list_per_page, list_max_show_all, list_editable, model_admin): self.model = model self.opts = model._meta self.lookup_opts = self.opts @@ -28,6 +30,7 @@ def __init__(self, request, model, list_display, list_display_links, list_filter self.search_fields = search_fields self.list_select_related = list_select_related self.list_per_page = list_per_page + self.list_max_show_all = list_max_show_all self.model_admin = model_admin # Get search parameters from the query string. @@ -45,12 +48,10 @@ def __init__(self, request, model, list_display, list_display_links, list_filter del self.params[ERROR_FLAG] self.list_editable = list_editable - self.order_field, self.order_type = self.get_ordering() self.query = request.GET.get(SEARCH_VAR, '') - self.query_set = self.get_query_set() + self.query_set = self.get_query_set(request) self.get_results(request) self.title = (self.is_popup and ugettext('Select %s') % force_unicode(self.opts.verbose_name) or ugettext('Select %s to change') % force_unicode(self.opts.verbose_name)) - self.filter_specs, self.has_filters = self.get_filters(request) self.pk_attname = self.lookup_opts.pk.attname From a6cb4dd7dfbb02f2b702f73b763d4629b9e74524 Mon Sep 17 00:00:00 2001 From: Corey Oordt Date: Sun, 24 Mar 2013 19:17:43 -0500 Subject: [PATCH 4/9] Fixed missing exclusions based on the setup of the settings. --- supertagging/modules/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supertagging/modules/__init__.py b/supertagging/modules/__init__.py index de152b3..0eb4f56 100644 --- a/supertagging/modules/__init__.py +++ b/supertagging/modules/__init__.py @@ -255,7 +255,7 @@ def _processEntities(field, data, obj, ctype, process_type, tags, date): stype = entity.pop('_type', '') # if type is in EXLCUSIONS, continue to next item. - if stype.lower() in map(lambda s: s.lower(), settings.EXCLUSIONS): + if stype.lower() in map(lambda s: s.lower(), settings.EXCLUSIONS['TAG_TYPE_EXCLUSIONS']): continue display_name = entity.pop('name', '') From 7e37e0c5133c12af40e333e41d311aa3f813acb0 Mon Sep 17 00:00:00 2001 From: Corey Oordt Date: Sun, 24 Mar 2013 19:18:17 -0500 Subject: [PATCH 5/9] Adding the STATIC_URL to the template --- .../supertagging/supertaggeditem/change_list.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/supertagging/templates/admin/supertagging/supertaggeditem/change_list.html b/supertagging/templates/admin/supertagging/supertaggeditem/change_list.html index 63227b2..2266a53 100644 --- a/supertagging/templates/admin/supertagging/supertaggeditem/change_list.html +++ b/supertagging/templates/admin/supertagging/supertaggeditem/change_list.html @@ -1,14 +1,14 @@ {% extends "admin/base_site.html" %} -{% load adminmedia admin_list i18n %} +{% load admin_list i18n %} {% block extrastyle %} {{ block.super }} - + {% if cl.formset %} - + {% endif %} {% if cl.formset or action_form %} - {% url admin:jsi18n as jsi18nurl %} + {% url "admin:jsi18n" as jsi18nurl %} {% endif %} {{ media.css }} @@ -43,11 +43,11 @@ {% trans "Home" %} - › + › {{ app_label|capfirst }} - › + › {{ cl.opts.verbose_name_plural|capfirst }} {% endblock %} From 7eb118794c8540330361487e06d46a09ec3e830c Mon Sep 17 00:00:00 2001 From: Corey Oordt Date: Sun, 24 Mar 2013 19:18:44 -0500 Subject: [PATCH 6/9] suffix was getting set to None, so made sure it was something! --- supertagging/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/supertagging/utils.py b/supertagging/utils.py index 88891eb..9e5eb53 100644 --- a/supertagging/utils.py +++ b/supertagging/utils.py @@ -373,7 +373,7 @@ def retrieve_freebase_desc(name, stype): ################ # Render Utils # ################ -def render_item(item, stype, template, suffix, template_path='supertagging/render', context={}): +def render_item(item, stype, template, suffix="", template_path='supertagging/render', context={}): """ Use to render tags, relations, tagged items and tagger relations. """ @@ -384,7 +384,7 @@ def render_item(item, stype, template, suffix, template_path='supertagging/rende app = item.content_type.app_label.lower() tp = "%s/%s" % (template_path, (stype or "")) - + suffix = suffix or "" # Retreive the template passed in # Retrieve the template based off of type and the content object with a suffix # Retrieve the template based off of type and the content object From 55437b95dad1c02ee6930fcef0e9ec0b49c81aeb Mon Sep 17 00:00:00 2001 From: Corey Oordt Date: Mon, 22 Apr 2013 07:03:35 -0500 Subject: [PATCH 7/9] Added only_installed=True to the get_model call which provides greater stability when getting models that haven't been loaded yet. --- supertagging/__init__.py | 2 +- supertagging/handlers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/supertagging/__init__.py b/supertagging/__init__.py index 6b4a12f..83ce429 100644 --- a/supertagging/__init__.py +++ b/supertagging/__init__.py @@ -3,7 +3,7 @@ 'minor': 6, 'micro': 0, 'releaselevel': 'beta', - 'serial': 1 + 'serial': 2 } diff --git a/supertagging/handlers.py b/supertagging/handlers.py index 8ac4c3a..cdc77e7 100644 --- a/supertagging/handlers.py +++ b/supertagging/handlers.py @@ -28,8 +28,8 @@ def setup_handlers(): try: for k, v in MODULES.items(): app_label, model_name = k.split('.') - model = get_model(app_label, model_name) + model = get_model(app_label, model_name, only_installed=False) # Add a tag attribute to the instance. if REGISTER_MODELS: register(model) From a1dbf374e0015e7106507dbdc021694e37636116 Mon Sep 17 00:00:00 2001 From: Corey Oordt Date: Mon, 22 Apr 2013 19:30:48 -0500 Subject: [PATCH 8/9] Updating the get_version method --- supertagging/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/supertagging/__init__.py b/supertagging/__init__.py index 83ce429..eb9e2fb 100644 --- a/supertagging/__init__.py +++ b/supertagging/__init__.py @@ -7,13 +7,13 @@ } -def get_version(): +def get_version(short=False): + assert __version_info__['releaselevel'] in ('alpha', 'beta', 'final') vers = ["%(major)i.%(minor)i" % __version_info__, ] - - if __version_info__['micro']: + if __version_info__['micro'] and not short: vers.append(".%(micro)i" % __version_info__) - if __version_info__['releaselevel'] != 'final': - vers.append('%(releaselevel)s%(serial)i' % __version_info__) + if __version_info__['releaselevel'] != 'final' and not short: + vers.append('%s%i' % (__version_info__['releaselevel'][0], __version_info__['serial'])) return ''.join(vers) __version__ = get_version() From ae8a5674fc23bfa4c2bd8a698fa8c11ef1fef42b Mon Sep 17 00:00:00 2001 From: Corey Oordt Date: Wed, 3 Jul 2013 15:17:49 -0500 Subject: [PATCH 9/9] Fixed a couple of old-labeled settings --- supertagging/__init__.py | 2 +- supertagging/handlers.py | 6 +++--- supertagging/markup.py | 4 ++-- supertagging/modules/__init__.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/supertagging/__init__.py b/supertagging/__init__.py index eb9e2fb..112293a 100644 --- a/supertagging/__init__.py +++ b/supertagging/__init__.py @@ -3,7 +3,7 @@ 'minor': 6, 'micro': 0, 'releaselevel': 'beta', - 'serial': 2 + 'serial': 3 } diff --git a/supertagging/handlers.py b/supertagging/handlers.py index cdc77e7..e202478 100644 --- a/supertagging/handlers.py +++ b/supertagging/handlers.py @@ -2,7 +2,7 @@ from django.db.models.signals import post_save, post_delete from supertagging.settings import (USE_QUEUE, MODULES, AUTO_PROCESS, ST_DEBUG, - MARKUP, MARKUP_FIELD_SUFFIX, REGISTER_MODELS) + MARKUP, REGISTER_MODELS) from supertagging import register @@ -39,7 +39,7 @@ def setup_handlers(): post_save.connect(save_handler, sender=model) post_delete.connect(delete_handler, sender=model) - if MARKUP: + if MARKUP['ENABLED']: # Add a custom attribute to a model with a marked up # version of the field specified. for f in v.get('fields', []): @@ -48,7 +48,7 @@ def setup_handlers(): if markup and MARKUP and field: from supertagging.markup import get_handler_module handler = get_handler_module(f.get('markup_handler', None)) - nfield = "%s__%s" % (field, MARKUP_FIELD_SUFFIX) + nfield = "%s__%s" % (field, MARKUP['FIELD_SUFFIX']) setattr(model, nfield, handler(model, field)) except Exception, e: diff --git a/supertagging/markup.py b/supertagging/markup.py index 4728ecb..ab091d8 100644 --- a/supertagging/markup.py +++ b/supertagging/markup.py @@ -30,7 +30,7 @@ def __get__(self, instance, owner): if settings.ST_DEBUG: raise Exception(e) - cache.set(self._get_cache_key(instance), data, settings.MARKUP_CONTENT_CACHE_TIMEOUT) + cache.set(self._get_cache_key(instance), data, settings.MARKUP['CONTENT_CACHE_TIMEOUT']) return data def _get_cache_key(self, instance=None): @@ -133,7 +133,7 @@ def markup_content(obj, field, markup_template='supertagging/markup.html'): for n, i in enumerate(full): if 'offset' in i and 'length' in i and 'exact' in i: off, le, act_val = i['offset'], i['length'], i['exact'] - if act_val.lower() in settings.MARKUP_EXCLUDES: + if act_val.lower() in settings.MARKUP['EXCLUDE']: continue else: continue diff --git a/supertagging/modules/__init__.py b/supertagging/modules/__init__.py index 0eb4f56..1d6ed19 100644 --- a/supertagging/modules/__init__.py +++ b/supertagging/modules/__init__.py @@ -198,7 +198,7 @@ def process(obj, tags=[]): processed_tags.extend(topics) processed_tags.extend(socialtags) - if settings.MARKUP: + if settings.MARKUP['ENABLED']: invalidate_markup_cache(obj, field) except Exception, e: