From 9cbb247dfb4faa47617438770bb6135155fade16 Mon Sep 17 00:00:00 2001 From: Stephen Burrows Date: Tue, 10 May 2011 14:21:02 -0400 Subject: [PATCH] Moved efficient QuerySetMappers into philo.utils and replaced them with a more complex/differentiated implementation that allows flexibility in terms of cacheing vs. lazy fetching and tree vs passthrough vs ordinary. Passthrough accepts any number of entities, and all Mappers support __setitem__. (Resolves issue #133.) Added a set_value method to Attributes. --- philo/models/base.py | 116 +++++--------------- philo/models/fields/entities.py | 15 +-- philo/models/nodes.py | 8 +- philo/utils.py | 186 ++++++++++++++++++++++++++++++-- 4 files changed, 213 insertions(+), 112 deletions(-) diff --git a/philo/models/base.py b/philo/models/base.py index 5d8fdc2..0ab9b70 100644 --- a/philo/models/base.py +++ b/philo/models/base.py @@ -1,5 +1,3 @@ -from UserDict import DictMixin - from django import forms from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes import generic @@ -13,7 +11,7 @@ from mptt.models import MPTTModel, MPTTModelBase, MPTTOptions from philo.exceptions import AncestorDoesNotExist from philo.models.fields import JSONField from philo.signals import entity_class_prepared -from philo.utils import ContentTypeRegistryLimiter, ContentTypeSubclassLimiter +from philo.utils import ContentTypeRegistryLimiter, ContentTypeSubclassLimiter, AttributeMapper, TreeAttributeMapper from philo.validators import json_validator @@ -262,35 +260,26 @@ class Attribute(models.Model): def __unicode__(self): return u'"%s": %s' % (self.key, self.value) + def set_value(self, value, value_class=JSONValue): + """Given a value and a value class, sets up self.value appropriately.""" + if isinstance(self.value, value_class): + val = self.value + else: + if isinstance(self.value, models.Model): + self.value.delete() + val = value_class() + + val.set_value(value) + val.save() + + self.value = val + self.save() + class Meta: app_label = 'philo' unique_together = (('key', 'entity_content_type', 'entity_object_id'), ('value_content_type', 'value_object_id')) -class QuerySetMapper(object, DictMixin): - def __init__(self, queryset, passthrough=None): - self.queryset = queryset - self.passthrough = passthrough - - def __getitem__(self, key): - try: - value = self.queryset.get(key__exact=key).value - except ObjectDoesNotExist: - if self.passthrough is not None: - return self.passthrough.__getitem__(key) - raise KeyError - else: - if value is not None: - return value.value - return value - - def keys(self): - keys = set(self.queryset.values_list('key', flat=True).distinct()) - if self.passthrough is not None: - keys |= set(self.passthrough.keys()) - return list(keys) - - class EntityOptions(object): def __init__(self, options): if options is not None: @@ -312,58 +301,15 @@ class EntityBase(models.base.ModelBase): return new -class EntityAttributeMapper(object, DictMixin): - def __init__(self, entity): - self.entity = entity - - def get_attributes(self): - return self.entity.attribute_set.all() - - def make_cache(self): - attributes = self.get_attributes() - value_lookups = {} - - for a in attributes: - value_lookups.setdefault(a.value_content_type, []).append(a.value_object_id) - - values_bulk = {} - - for ct, pks in value_lookups.items(): - values_bulk[ct] = ct.model_class().objects.in_bulk(pks) - - self._cache = dict([(a.key, getattr(values_bulk[a.value_content_type].get(a.value_object_id), 'value', None)) for a in attributes]) - - def __getitem__(self, key): - if not hasattr(self, '_cache'): - self.make_cache() - return self._cache[key] - - def keys(self): - if not hasattr(self, '_cache'): - self.make_cache() - return self._cache.keys() - - def items(self): - if not hasattr(self, '_cache'): - self.make_cache() - return self._cache.items() - - def values(self): - if not hasattr(self, '_cache'): - self.make_cache() - return self._cache.values() - - class Entity(models.Model): """An abstract class that simplifies access to related attributes. Most models provided by Philo subclass Entity.""" __metaclass__ = EntityBase attribute_set = generic.GenericRelation(Attribute, content_type_field='entity_content_type', object_id_field='entity_object_id') - @property - def attributes(self): + def get_attribute_mapper(self, mapper=AttributeMapper): """ - Property that returns a dictionary-like object which can be used to retrieve related :class:`Attribute`\ s' values directly. + Returns a dictionary-like object which can be used to retrieve related :class:`Attribute`\ s' values directly. Example:: @@ -374,7 +320,8 @@ class Entity(models.Model): u'eggs' """ - return EntityAttributeMapper(self) + return mapper(self) + attributes = property(get_attribute_mapper) class Meta: abstract = True @@ -536,22 +483,14 @@ class TreeEntityBase(MPTTModelBase, EntityBase): return meta.register(cls) -class TreeEntityAttributeMapper(EntityAttributeMapper): - def get_attributes(self): - ancestors = dict(self.entity.get_ancestors(include_self=True).values_list('pk', 'level')) - ct = ContentType.objects.get_for_model(self.entity) - return sorted(Attribute.objects.filter(entity_content_type=ct, entity_object_id__in=ancestors.keys()), key=lambda x: ancestors[x.entity_object_id]) - - class TreeEntity(Entity, TreeModel): """An abstract subclass of Entity which represents a tree relationship.""" __metaclass__ = TreeEntityBase - @property - def attributes(self): + def get_attribute_mapper(self, mapper=None): """ - Property that returns a dictionary-like object which can be used to retrieve related :class:`Attribute`\ s' values directly. If an attribute with a given key is not related to the :class:`Entity`, then the object will check the parent's attributes. + Returns a dictionary-like object which can be used to retrieve related :class:`Attribute`\ s' values directly. If an attribute with a given key is not related to the :class:`Entity`, then the object will check the parent's attributes. Example:: @@ -564,10 +503,13 @@ class TreeEntity(Entity, TreeModel): u'eggs' """ - - if self.parent: - return TreeEntityAttributeMapper(self) - return super(TreeEntity, self).attributes + if mapper is None: + if self.parent: + mapper = TreeAttributeMapper + else: + mapper = AttributeMapper + return super(TreeEntity, self).get_attribute_mapper(mapper) + attributes = property(get_attribute_mapper) class Meta: abstract = True \ No newline at end of file diff --git a/philo/models/fields/entities.py b/philo/models/fields/entities.py index c37d496..3e96d13 100644 --- a/philo/models/fields/entities.py +++ b/philo/models/fields/entities.py @@ -130,20 +130,7 @@ def process_attribute_fields(sender, instance, created, **kwargs): attribute = Attribute() attribute.entity = instance attribute.key = field.attribute_key - - value_class = field.value_class - if isinstance(attribute.value, value_class): - value = attribute.value - else: - if isinstance(attribute.value, models.Model): - attribute.value.delete() - value = value_class() - - value.set_value(getattr(instance, field.name, None)) - value.save() - - attribute.value = value - attribute.save() + attribute.set_value(value=getattr(instance, field.name, None), value_class=field.value_class) del instance.__dict__[ATTRIBUTE_REGISTRY] diff --git a/philo/models/nodes.py b/philo/models/nodes.py index 2d4639f..44a4d31 100644 --- a/philo/models/nodes.py +++ b/philo/models/nodes.py @@ -12,9 +12,9 @@ from django.template import add_to_builtins as register_templatetags from django.utils.encoding import smart_str from philo.exceptions import MIDDLEWARE_NOT_CONFIGURED, ViewCanNotProvideSubpath, ViewDoesNotProvideSubpaths -from philo.models.base import TreeEntity, Entity, QuerySetMapper, register_value_model +from philo.models.base import TreeEntity, Entity, register_value_model from philo.models.fields import JSONField -from philo.utils import ContentTypeSubclassLimiter +from philo.utils import ContentTypeSubclassLimiter, LazyPassthroughAttributeMapper from philo.validators import RedirectValidator from philo.signals import view_about_to_render, view_finished_rendering @@ -176,10 +176,10 @@ class View(Entity): def attributes_with_node(self, node): """ - Returns a :class:`~philo.models.base.QuerySetMapper` using the :class:`Node`'s attributes as a passthrough. + Returns a dictionary-like object which can be used to directly retrieve the values of :class:`Attribute`\ s related to the :class:`View`, falling back on similar object which retrieves the values of the passed-in node and its ancestors. """ - return QuerySetMapper(self.attribute_set, passthrough=node.attributes) + return LazyPassthroughAttributeMapper((self, node)) def render_to_response(self, request, extra_context=None): """ diff --git a/philo/utils.py b/philo/utils.py index 57f949e..f055051 100644 --- a/philo/utils.py +++ b/philo/utils.py @@ -1,3 +1,5 @@ +from UserDict import DictMixin + from django.db import models from django.contrib.contenttypes.models import ContentType from django.core.paginator import Paginator, EmptyPage @@ -5,6 +7,17 @@ from django.template import Context from django.template.loader_tags import ExtendsNode, ConstantIncludeNode +def fattr(*args, **kwargs): + def wrapper(function): + for key in kwargs: + setattr(function, key, kwargs[key]) + return function + return wrapper + + +### ContentTypeLimiters + + class ContentTypeLimiter(object): def q_object(self): return models.Q(pk__in=[]) @@ -59,12 +72,7 @@ class ContentTypeSubclassLimiter(ContentTypeLimiter): return models.Q(pk__in=contenttype_pks) -def fattr(*args, **kwargs): - def wrapper(function): - for key in kwargs: - setattr(function, key, kwargs[key]) - return function - return wrapper +### Pagination def paginate(objects, per_page=None, page_number=1): @@ -107,6 +115,9 @@ def paginate(objects, per_page=None, page_number=1): return paginator, page, objects +### Facilitating template analysis. + + LOADED_TEMPLATE_ATTR = '_philo_loaded_template' BLANK_CONTEXT = Context() @@ -121,4 +132,165 @@ def get_included(self): # We ignore the IncludeNode because it will never work in a blank context. setattr(ExtendsNode, LOADED_TEMPLATE_ATTR, property(get_extended)) -setattr(ConstantIncludeNode, LOADED_TEMPLATE_ATTR, property(get_included)) \ No newline at end of file +setattr(ConstantIncludeNode, LOADED_TEMPLATE_ATTR, property(get_included)) + + +### AttributeMappers + + +class AttributeMapper(object, DictMixin): + def __init__(self, entity): + self.entity = entity + self.clear_cache() + + def __getitem__(self, key): + if not self._cache_populated: + self._populate_cache() + return self._cache[key] + + def __setitem__(self, key, value): + # Prevent circular import. + from philo.models.base import JSONValue, ForeignKeyValue, ManyToManyValue, Attribute + old_attr = self.get_attribute(key) + if old_attr and old_attr.entity_content_type == ContentType.objects.get_for_model(self.entity) and old_attr.entity_object_id == self.entity.pk: + attribute = old_attr + else: + attribute = Attribute(key=key) + attribute.entity = self.entity + attribute.full_clean() + + if isinstance(value, models.query.QuerySet): + value_class = ManyToManyValue + elif isinstance(value, models.Model): + value_class = ForeignKeyValue + else: + value_class = JSONValue + + attribute.set_value(value=value, value_class=value_class) + self._cache[key] = attribute.value.value + self._attributes_cache[key] = attribute + + def get_attributes(self): + return self.entity.attribute_set.all() + + def get_attribute(self, key): + if not self._cache_populated: + self._populate_cache() + return self._attributes_cache.get(key, None) + + def keys(self): + if not self._cache_populated: + self._populate_cache() + return self._cache.keys() + + def items(self): + if not self._cache_populated: + self._populate_cache() + return self._cache.items() + + def values(self): + if not self._cache_populated: + self._populate_cache() + return self._cache.values() + + def _populate_cache(self): + if self._cache_populated: + return + + attributes = self.get_attributes() + value_lookups = {} + + for a in attributes: + value_lookups.setdefault(a.value_content_type, []).append(a.value_object_id) + self._attributes_cache[a.key] = a + + values_bulk = {} + + for ct, pks in value_lookups.items(): + values_bulk[ct] = ct.model_class().objects.in_bulk(pks) + + self._cache.update(dict([(a.key, getattr(values_bulk[a.value_content_type].get(a.value_object_id), 'value', None)) for a in attributes])) + self._cache_populated = True + + def clear_cache(self): + self._cache = {} + self._attributes_cache = {} + self._cache_populated = False + + +class LazyAttributeMapperMixin(object): + def __getitem__(self, key): + if key not in self._cache and not self._cache_populated: + self._add_to_cache(key) + return self._cache[key] + + def get_attribute(self, key): + if key not in self._attributes_cache and not self._cache_populated: + self._add_to_cache(key) + return self._attributes_cache[key] + + def _add_to_cache(self, key): + try: + attr = self.get_attributes().get(key=key) + except Attribute.DoesNotExist: + raise KeyError + else: + val = getattr(attr.value, 'value', None) + self._cache[key] = val + self._attributes_cache[key] = attr + + +class LazyAttributeMapper(LazyAttributeMapperMixin, AttributeMapper): + def get_attributes(self): + return super(LazyAttributeMapper, self).get_attributes().exclude(key__in=self._cache.keys()) + + +class TreeAttributeMapper(AttributeMapper): + def get_attributes(self): + from philo.models import Attribute + ancestors = dict(self.entity.get_ancestors(include_self=True).values_list('pk', 'level')) + ct = ContentType.objects.get_for_model(self.entity) + return sorted(Attribute.objects.filter(entity_content_type=ct, entity_object_id__in=ancestors.keys()), key=lambda x: ancestors[x.entity_object_id]) + + +class LazyTreeAttributeMapper(LazyAttributeMapperMixin, TreeAttributeMapper): + def get_attributes(self): + return super(LazyTreeAttributeMapper, self).get_attributes().exclude(key__in=self._cache.keys()) + + +class PassthroughAttributeMapper(AttributeMapper): + def __init__(self, entities): + self._attributes = [e.attributes for e in entities] + super(PassthroughAttributeMapper, self).__init__(self._attributes[0].entity) + + def _populate_cache(self): + if self._cache_populated: + return + + for a in reversed(self._attributes): + a._populate_cache() + self._attributes_cache.update(a._attributes_cache) + self._cache.update(a._cache) + + self._cache_populated = True + + def get_attributes(self): + raise NotImplementedError + + def clear_cache(self): + super(PassthroughAttributeMapper, self).clear_cache() + for a in self._attributes: + a.clear_cache() + + +class LazyPassthroughAttributeMapper(LazyAttributeMapperMixin, PassthroughAttributeMapper): + def _add_to_cache(self, key): + for a in self._attributes: + try: + self._cache[key] = a[key] + self._attributes_cache[key] = a.get_attribute(key) + except KeyError: + pass + else: + break + return self._cache[key] \ No newline at end of file -- 2.20.1