forum/models/base.py

   1 import datetime
   2 import re
   3 try:
   4     from hashlib import md5
   5 except:
   6     from md5 import new as md5
   7 from urllib import quote_plus, urlencode
   8 from django.db import models, IntegrityError, connection, transaction
   9 from django.utils.http import urlquote  as django_urlquote
  10 from django.utils.html import strip_tags
  11 from django.conf import settings as django_settings
  12 from django.core.urlresolvers import reverse
  13 from django.contrib.contenttypes import generic
  14 from django.contrib.contenttypes.models import ContentType
  15 from django.core.cache import cache
  16 from django.template.defaultfilters import slugify
  17 from django.db.models.signals import post_delete, post_save, pre_save, pre_delete
  18 from django.utils.translation import ugettext as _
  19 from django.utils.safestring import mark_safe
  20 from django.utils.encoding import force_unicode
  21 from django.contrib.sitemaps import ping_google
  22 import django.dispatch
  23 from forum import settings
  24 import logging
  25
  26
  27 if not hasattr(cache, 'get_many'):
  28     #put django 1.2 code here
  29     pass
  30
  31 class LazyQueryList(object):
  32     def __init__(self, model, items):
  33         self.items = items
  34         self.model = model
  35
  36     def __getitem__(self, k):
  37         return self.model.objects.get(id=self.items[k][0])
  38
  39     def __iter__(self):
  40         for id in self.items:
  41             yield self.model.objects.get(id=id[0])
  42
  43     def __len__(self):
  44         return len(self.items)
  45
  46 class ToFetch(unicode):
  47     pass
  48
  49 class CachedQuerySet(models.query.QuerySet):
  50
  51     def lazy(self):
  52         if not len(self.query.aggregates):
  53             values_list = ['id']
  54
  55             if len(self.query.extra):
  56                 extra_keys = self.query.extra.keys()
  57                 values_list += extra_keys
  58
  59             return LazyQueryList(self.model, list(self.values_list(*values_list)))
  60         else:
  61             return self
  62
  63     def obj_from_datadict(self, datadict):
  64         obj = self.model()
  65         obj.__dict__.update(datadict)
  66
  67         if hasattr(obj, '_state'):
  68             obj._state.db = 'default'
  69
  70         return obj
  71
  72     def _base_clone(self):
  73         return self._clone(klass=models.query.QuerySet)
  74
  75     def get(self, *args, **kwargs):
  76         key = self.model.infer_cache_key(kwargs)
  77
  78         if key is not None:
  79             obj = cache.get(key)
  80
  81             if obj is None:
  82                 obj = self._base_clone().get(*args, **kwargs)
  83                 obj.cache()
  84             else:
  85                 obj = self.obj_from_datadict(obj)
  86
  87             obj.reset_original_state()
  88
  89             return obj
  90
  91         return self._base_clone().get(*args, **kwargs)
  92
  93     def _fetch_from_query_cache(self, key):
  94         invalidation_key = self.model._get_cache_query_invalidation_key()
  95         cached_result = cache.get_many([invalidation_key, key])
  96
  97         if not invalidation_key in cached_result:
  98             self.model._set_query_cache_invalidation_timestamp()
  99             return None
 100
 101         if (key in cached_result) and(cached_result[invalidation_key] < cached_result[key][0]):
 102             return cached_result[key][1]
 103
 104         return None
 105
 106     def count(self):
 107         cache_key = self.model._generate_cache_key("CNT:%s" % self._get_query_hash())
 108         result = self._fetch_from_query_cache(cache_key)
 109
 110         if result is not None:
 111             return result
 112
 113         result = super(CachedQuerySet, self).count()
 114         cache.set(cache_key, (datetime.datetime.now(), result), 60 * 60)
 115         return result
 116
 117     def iterator(self):
 118         cache_key = self.model._generate_cache_key("QUERY:%s" % self._get_query_hash())
 119         on_cache_query_attr = self.model.value_to_list_on_cache_query()
 120
 121         to_return = None
 122         to_cache = {}
 123
 124         with_aggregates = len(self.query.aggregates) > 0
 125         key_list = self._fetch_from_query_cache(cache_key)
 126
 127         if key_list is None:
 128             if not with_aggregates:
 129                 values_list = [on_cache_query_attr]
 130
 131                 if len(self.query.extra):
 132                     values_list += self.query.extra.keys()
 133
 134                 key_list = [v[0] for v in self.values_list(*values_list)]
 135                 to_cache[cache_key] = (datetime.datetime.now(), key_list)
 136             else:
 137                 to_return = list(super(CachedQuerySet, self).iterator())
 138                 to_cache[cache_key] = (datetime.datetime.now(), [
 139                     (row.__dict__[on_cache_query_attr], dict([(k, row.__dict__[k]) for k in self.query.aggregates.keys()]))
 140                     for row in to_return])
 141         elif with_aggregates:
 142             tmp = key_list
 143             key_list = [k[0] for k in tmp]
 144             with_aggregates = [k[1] for k in tmp]
 145             del tmp
 146
 147         if (not to_return) and key_list:
 148             row_keys = [self.model.infer_cache_key({on_cache_query_attr: attr}) for attr in key_list]
 149             cached = cache.get_many(row_keys)
 150
 151             to_return = [
 152                 (ck in cached) and self.obj_from_datadict(cached[ck]) or ToFetch(force_unicode(key_list[i])) for i, ck in enumerate(row_keys)
 153             ]
 154
 155             if len(cached) != len(row_keys):
 156                 to_fetch = [unicode(tr) for tr in to_return if isinstance(tr, ToFetch)]
 157
 158                 fetched = dict([(force_unicode(r.__dict__[on_cache_query_attr]), r) for r in
 159                               models.query.QuerySet(self.model).filter(**{"%s__in" % on_cache_query_attr: to_fetch})])
 160
 161                 to_return = [(isinstance(tr, ToFetch) and fetched[unicode(tr)] or tr) for tr in to_return]
 162                 to_cache.update(dict([(self.model.infer_cache_key({on_cache_query_attr: attr}), r._as_dict()) for attr, r in fetched.items()]))
 163
 164             if with_aggregates:
 165                 for i, r in enumerate(to_return):
 166                     r.__dict__.update(with_aggregates[i])
 167
 168
 169         if len(to_cache):
 170             cache.set_many(to_cache, 60 * 60)
 171
 172         if to_return:
 173             for row in to_return:
 174                 if hasattr(row, 'leaf'):
 175                     row = row.leaf
 176
 177                 row.reset_original_state()
 178                 yield row
 179
 180     def _get_query_hash(self):
 181         return md5(unicode(self.query).encode("utf-8")).hexdigest()
 182
 183
 184
 185 class CachedManager(models.Manager):
 186     use_for_related_fields = True
 187
 188     def get_query_set(self):
 189         return CachedQuerySet(self.model)
 190
 191     def get_or_create(self, *args, **kwargs):
 192         try:
 193             return self.get(*args, **kwargs)
 194         except:
 195             return super(CachedManager, self).get_or_create(*args, **kwargs)
 196
 197
 198 class DenormalizedField(object):
 199     def __init__(self, manager, *args, **kwargs):
 200         self.manager = manager
 201         self.filter = (args, kwargs)
 202
 203     def setup_class(self, cls, name):
 204         dict_name = '_%s_dencache_' % name
 205
 206         def getter(inst):
 207             val = inst.__dict__.get(dict_name, None)
 208
 209             if val is None:
 210                 val = getattr(inst, self.manager).filter(*self.filter[0], **self.filter[1]).count()
 211                 inst.__dict__[dict_name] = val
 212                 inst.cache()
 213
 214             return val
 215
 216         def reset_cache(inst):
 217             inst.__dict__.pop(dict_name, None)
 218             inst.uncache()
 219
 220         cls.add_to_class(name, property(getter))
 221         cls.add_to_class("reset_%s_cache" % name, reset_cache)
 222
 223
 224 class BaseMetaClass(models.Model.__metaclass__):
 225     to_denormalize = []
 226
 227     def __new__(cls, *args, **kwargs):
 228         new_cls = super(BaseMetaClass, cls).__new__(cls, *args, **kwargs)
 229
 230         BaseMetaClass.to_denormalize.extend(
 231             [(new_cls, name, field) for name, field in new_cls.__dict__.items() if isinstance(field, DenormalizedField)]
 232         )
 233
 234         return new_cls
 235
 236     @classmethod
 237     def setup_denormalizes(cls):
 238         for new_cls, name, field in BaseMetaClass.to_denormalize:
 239             field.setup_class(new_cls, name)
 240
 241
 242 class BaseModel(models.Model):
 243     __metaclass__ = BaseMetaClass
 244
 245     objects = CachedManager()
 246
 247     class Meta:
 248         abstract = True
 249         app_label = 'forum'
 250
 251     def __init__(self, *args, **kwargs):
 252         super(BaseModel, self).__init__(*args, **kwargs)
 253         self.reset_original_state(kwargs.keys())
 254
 255     def reset_original_state(self, reset_fields=None):
 256         self._original_state = self._as_dict()
 257
 258         if reset_fields:
 259             self._original_state.update(dict([(f, None) for f in reset_fields]))
 260
 261     def get_dirty_fields(self):
 262         return [f.name for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]]
 263
 264     def _as_dict(self):
 265         return dict([(name, getattr(self, name)) for name in
 266                      ([f.attname for f in self._meta.fields] + [k for k in self.__dict__.keys() if k.endswith('_dencache_')])
 267         ])
 268
 269     def _get_update_kwargs(self):
 270         return dict([
 271             (f.name, getattr(self, f.name)) for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]
 272         ])
 273
 274     def save(self, full_save=False, *args, **kwargs):
 275         put_back = [k for k, v in self.__dict__.items() if isinstance(v, models.expressions.ExpressionNode)]
 276
 277         if hasattr(self, '_state'):
 278             self._state.db = 'default'
 279
 280         if self.id and not full_save:
 281             self.__class__.objects.filter(id=self.id).update(**self._get_update_kwargs())
 282         else:
 283             super(BaseModel, self).save()
 284
 285         if put_back:
 286             try:
 287                 self.__dict__.update(
 288                     self.__class__.objects.filter(id=self.id).values(*put_back)[0]
 289                 )
 290             except:
 291                 logging.error("Unable to read %s from %s" % (", ".join(put_back), self.__class__.__name__))
 292                 self.uncache()
 293
 294         self.reset_original_state()
 295         self._set_query_cache_invalidation_timestamp()
 296         self.cache()
 297
 298     @classmethod
 299     def _get_cache_query_invalidation_key(cls):
 300         return cls._generate_cache_key("INV_TS")
 301
 302     @classmethod
 303     def _set_query_cache_invalidation_timestamp(cls):
 304         cache.set(cls._get_cache_query_invalidation_key(), datetime.datetime.now(), 60 * 60 * 24)
 305
 306         for base in filter(lambda c: issubclass(c, BaseModel) and (not c is BaseModel), cls.__bases__):
 307             base._set_query_cache_invalidation_timestamp()
 308
 309     @classmethod
 310     def _generate_cache_key(cls, key, group=None):
 311         if group is None:
 312             group = cls.__name__
 313
 314         return '%s:%s:%s' % (settings.APP_URL, group, key)
 315
 316     def cache_key(self):
 317         return self._generate_cache_key(self.id)
 318
 319     @classmethod
 320     def value_to_list_on_cache_query(cls):
 321         return 'id'
 322
 323     @classmethod
 324     def infer_cache_key(cls, querydict):
 325         try:
 326             pk = [v for (k,v) in querydict.items() if k in ('pk', 'pk__exact', 'id', 'id__exact'
 327                             ) or k.endswith('_ptr__pk') or k.endswith('_ptr__id')][0]
 328
 329             cache_key = cls._generate_cache_key(pk)
 330
 331             if len(cache_key) > django_settings.CACHE_MAX_KEY_LENGTH:
 332                 cache_key = cache_key[:django_settings.CACHE_MAX_KEY_LENGTH]
 333
 334             return cache_key
 335         except:
 336             return None
 337
 338     def cache(self):
 339         cache.set(self.cache_key(), self._as_dict(), 60 * 60)
 340
 341     def uncache(self):
 342         cache.delete(self.cache_key())
 343
 344     def delete(self):
 345         self.uncache()
 346         self._set_query_cache_invalidation_timestamp()
 347         super(BaseModel, self).delete()
 348
 349
 350 from user import User
 351 from node import Node, NodeRevision, NodeManager
 352 from action import Action
 353
 354
 355
 356
 357