]> git.openstreetmap.org Git - osqa.git/blob - forum/models/base.py
912c21ac768129ce66b8d955b5ee751e4869f3ea
[osqa.git] / forum / models / base.py
1 import datetime
2 import re
3 try:
4     from hashlib import md5
5 except:
6     from md5 import new as md5
7 from urllib import quote_plus, urlencode
8 from django.db import models, IntegrityError, connection, transaction
9 from django.utils.http import urlquote  as django_urlquote
10 from django.utils.html import strip_tags
11 from django.conf import settings as django_settings
12 from django.core.urlresolvers import reverse
13 from django.contrib.contenttypes import generic
14 from django.contrib.contenttypes.models import ContentType
15 from django.core.cache import cache
16 from django.template.defaultfilters import slugify
17 from django.db.models.signals import post_delete, post_save, pre_save, pre_delete
18 from django.utils.translation import ugettext as _
19 from django.utils.safestring import mark_safe
20 from django.utils.encoding import force_unicode
21 from django.contrib.sitemaps import ping_google
22 import django.dispatch
23 from forum import settings
24 import logging
25
26
27 if not hasattr(cache, 'get_many'):
28     #put django 1.2 code here
29     pass
30
31 class LazyQueryList(object):
32     def __init__(self, model, items):
33         self.items = items
34         self.model = model
35
36     def __getitem__(self, k):
37         return self.model.objects.get(id=self.items[k][0])
38
39     def __iter__(self):
40         for id in self.items:
41             yield self.model.objects.get(id=id[0])
42
43     def __len__(self):
44         return len(self.items)
45
46 class ToFetch(unicode):
47     pass
48
49 class CachedQuerySet(models.query.QuerySet):
50
51     def lazy(self):
52         if not len(self.query.aggregates):
53             values_list = ['id']
54
55             if len(self.query.extra):
56                 extra_keys = self.query.extra.keys()
57                 values_list += extra_keys
58
59             return LazyQueryList(self.model, list(self.values_list(*values_list)))
60         else:
61             return self
62
63     def obj_from_datadict(self, datadict):
64         obj = self.model()
65         obj.__dict__.update(datadict)
66
67         if hasattr(obj, '_state'):
68             obj._state.db = 'default'
69
70         return obj
71
72     def _base_clone(self):
73         return self._clone(klass=models.query.QuerySet)
74
75     def get(self, *args, **kwargs):
76         key = self.model.infer_cache_key(kwargs)
77
78         if key is not None:
79             obj = cache.get(key)
80
81             if obj is None:
82                 obj = self._base_clone().get(*args, **kwargs)
83                 obj.cache()
84             else:
85                 obj = self.obj_from_datadict(obj)
86
87             obj.reset_original_state()
88
89             return obj
90
91         return self._base_clone().get(*args, **kwargs)
92
93     def _fetch_from_query_cache(self, key):
94         invalidation_key = self.model._get_cache_query_invalidation_key()
95         cached_result = cache.get_many([invalidation_key, key])
96
97         if not invalidation_key in cached_result:
98             self.model._set_query_cache_invalidation_timestamp()
99             return None
100
101         if (key in cached_result) and(cached_result[invalidation_key] < cached_result[key][0]):
102             return cached_result[key][1]
103
104         return None
105
106     def count(self):
107         cache_key = self.model._generate_cache_key("CNT:%s" % self._get_query_hash())
108         result = self._fetch_from_query_cache(cache_key)
109
110         if result is not None:
111             return result
112
113         result = super(CachedQuerySet, self).count()
114         cache.set(cache_key, (datetime.datetime.now(), result), 60 * 60)
115         return result
116
117     def iterator(self):
118         cache_key = self.model._generate_cache_key("QUERY:%s" % self._get_query_hash())
119         on_cache_query_attr = self.model.value_to_list_on_cache_query()
120
121         to_return = None
122         to_cache = {}
123
124         with_aggregates = len(self.query.aggregates) > 0
125         key_list = self._fetch_from_query_cache(cache_key)
126
127         if key_list is None:
128             if not with_aggregates:
129                 values_list = [on_cache_query_attr]
130
131                 if len(self.query.extra):
132                     values_list += self.query.extra.keys()
133
134                 key_list = [v[0] for v in self.values_list(*values_list)]
135                 to_cache[cache_key] = (datetime.datetime.now(), key_list)
136             else:
137                 to_return = list(super(CachedQuerySet, self).iterator())
138                 to_cache[cache_key] = (datetime.datetime.now(), [
139                     (row.__dict__[on_cache_query_attr], dict([(k, row.__dict__[k]) for k in self.query.aggregates.keys()]))
140                     for row in to_return])
141         elif with_aggregates:
142             tmp = key_list
143             key_list = [k[0] for k in tmp]
144             with_aggregates = [k[1] for k in tmp]
145             del tmp
146
147         if (not to_return) and key_list:
148             row_keys = [self.model.infer_cache_key({on_cache_query_attr: attr}) for attr in key_list]
149             cached = cache.get_many(row_keys)
150
151             to_return = [
152                 (ck in cached) and self.obj_from_datadict(cached[ck]) or ToFetch(force_unicode(key_list[i])) for i, ck in enumerate(row_keys)
153             ]
154
155             if len(cached) != len(row_keys):
156                 to_fetch = [unicode(tr) for tr in to_return if isinstance(tr, ToFetch)]
157
158                 fetched = dict([(force_unicode(r.__dict__[on_cache_query_attr]), r) for r in
159                               models.query.QuerySet(self.model).filter(**{"%s__in" % on_cache_query_attr: to_fetch})])
160
161                 to_return = [(isinstance(tr, ToFetch) and fetched[unicode(tr)] or tr) for tr in to_return]
162                 to_cache.update(dict([(self.model.infer_cache_key({on_cache_query_attr: attr}), r._as_dict()) for attr, r in fetched.items()]))
163
164             if with_aggregates:
165                 for i, r in enumerate(to_return):
166                     r.__dict__.update(with_aggregates[i])
167
168
169         if len(to_cache):
170             cache.set_many(to_cache, 60 * 60)
171
172         if to_return:
173             for row in to_return:
174                 if hasattr(row, 'leaf'):
175                     row = row.leaf
176
177                 row.reset_original_state()
178                 yield row
179
180     def _get_query_hash(self):
181         try:
182             return md5(unicode(self.query).encode("utf-8")).hexdigest()
183         except:
184             try:
185                 return md5(self.query).hexdigest()
186             except:
187                 return md5(str(self.query)).hexdigest()
188
189
190 class CachedManager(models.Manager):
191     use_for_related_fields = True
192
193     def get_query_set(self):
194         return CachedQuerySet(self.model)
195
196     def get_or_create(self, *args, **kwargs):
197         try:
198             return self.get(*args, **kwargs)
199         except:
200             return super(CachedManager, self).get_or_create(*args, **kwargs)
201
202
203 class DenormalizedField(object):
204     def __init__(self, manager, *args, **kwargs):
205         self.manager = manager
206         self.filter = (args, kwargs)
207
208     def setup_class(self, cls, name):
209         dict_name = '_%s_dencache_' % name
210
211         def getter(inst):
212             val = inst.__dict__.get(dict_name, None)
213
214             if val is None:
215                 val = getattr(inst, self.manager).filter(*self.filter[0], **self.filter[1]).count()
216                 inst.__dict__[dict_name] = val
217                 inst.cache()
218
219             return val
220
221         def reset_cache(inst):
222             inst.__dict__.pop(dict_name, None)
223             inst.uncache()
224
225         cls.add_to_class(name, property(getter))
226         cls.add_to_class("reset_%s_cache" % name, reset_cache)
227
228
229 class BaseMetaClass(models.Model.__metaclass__):
230     to_denormalize = []
231
232     def __new__(cls, *args, **kwargs):
233         new_cls = super(BaseMetaClass, cls).__new__(cls, *args, **kwargs)
234
235         BaseMetaClass.to_denormalize.extend(
236             [(new_cls, name, field) for name, field in new_cls.__dict__.items() if isinstance(field, DenormalizedField)]
237         )
238
239         return new_cls
240
241     @classmethod
242     def setup_denormalizes(cls):
243         for new_cls, name, field in BaseMetaClass.to_denormalize:
244             field.setup_class(new_cls, name)
245
246
247 class BaseModel(models.Model):
248     __metaclass__ = BaseMetaClass
249
250     objects = CachedManager()
251
252     class Meta:
253         abstract = True
254         app_label = 'forum'
255
256     def __init__(self, *args, **kwargs):
257         super(BaseModel, self).__init__(*args, **kwargs)
258         self.reset_original_state(kwargs.keys())
259
260     def reset_original_state(self, reset_fields=None):
261         self._original_state = self._as_dict()
262         
263         if reset_fields:
264             self._original_state.update(dict([(f, None) for f in reset_fields]))
265
266     def get_dirty_fields(self):
267         return [f.name for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]]
268
269     def _as_dict(self):
270         return dict([(name, getattr(self, name)) for name in
271                      ([f.attname for f in self._meta.fields] + [k for k in self.__dict__.keys() if k.endswith('_dencache_')])
272         ])
273
274     def _get_update_kwargs(self):
275         return dict([
276             (f.name, getattr(self, f.name)) for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]
277         ])
278
279     def save(self, full_save=False, *args, **kwargs):
280         put_back = [k for k, v in self.__dict__.items() if isinstance(v, models.expressions.ExpressionNode)]
281
282         if hasattr(self, '_state'):
283             self._state.db = 'default'
284
285         if self.id and not full_save:
286             self.__class__.objects.filter(id=self.id).update(**self._get_update_kwargs())
287         else:
288             super(BaseModel, self).save()
289
290         if put_back:
291             try:
292                 self.__dict__.update(
293                     self.__class__.objects.filter(id=self.id).values(*put_back)[0]
294                 )
295             except:
296                 logging.error("Unable to read %s from %s" % (", ".join(put_back), self.__class__.__name__))
297                 self.uncache()
298
299         self.reset_original_state()
300         self._set_query_cache_invalidation_timestamp()
301         self.cache()
302
303     @classmethod
304     def _get_cache_query_invalidation_key(cls):
305         return cls._generate_cache_key("INV_TS")
306
307     @classmethod
308     def _set_query_cache_invalidation_timestamp(cls):
309         cache.set(cls._get_cache_query_invalidation_key(), datetime.datetime.now(), 60 * 60 * 24)
310
311         for base in filter(lambda c: issubclass(c, BaseModel) and (not c is BaseModel), cls.__bases__):
312             base._set_query_cache_invalidation_timestamp()
313
314     @classmethod
315     def _generate_cache_key(cls, key, group=None):
316         if group is None:
317             group = cls.__name__
318
319         return '%s:%s:%s' % (settings.APP_URL, group, key)
320
321     def cache_key(self):
322         return self._generate_cache_key(self.id)
323
324     @classmethod
325     def value_to_list_on_cache_query(cls):
326         return 'id'
327
328     @classmethod
329     def infer_cache_key(cls, querydict):
330         try:
331             pk = [v for (k,v) in querydict.items() if k in ('pk', 'pk__exact', 'id', 'id__exact'
332                             ) or k.endswith('_ptr__pk') or k.endswith('_ptr__id')][0]
333
334             cache_key = cls._generate_cache_key(pk)
335
336             if len(cache_key) > django_settings.CACHE_MAX_KEY_LENGTH:
337                 cache_key = cache_key[:django_settings.CACHE_MAX_KEY_LENGTH]
338
339             return cache_key
340         except:
341             return None
342
343     def cache(self):
344         cache.set(self.cache_key(), self._as_dict(), 60 * 60)
345
346     def uncache(self):
347         cache.delete(self.cache_key())
348
349     def delete(self):
350         self.uncache()
351         self._set_query_cache_invalidation_timestamp()
352         super(BaseModel, self).delete()
353
354
355 from user import User
356 from node import Node, NodeRevision, NodeManager
357 from action import Action
358
359
360
361
362