--- /dev/null
+#encoding: utf-8
+
+from django.conf import settings
+from django.contrib.sites.models import Site
+from django.core.cache import cache
+from django.db.models.options import get_verbose_name as convert_camelcase
+from django.utils import simplejson as json
+from django.utils.http import urlquote_plus
+from django.utils.safestring import mark_safe
+from django.utils.text import capfirst
+from django.template import loader, Context, Template
+import datetime
+from philo.contrib.sobol.utils import make_tracking_querydict
+
+try:
+ from eventlet.green import urllib2
+except:
+ import urllib2
+
+
+__all__ = (
+ 'Result', 'BaseSearch', 'DatabaseSearch', 'URLSearch', 'JSONSearch', 'GoogleSearch', 'registry'
+)
+
+
+SEARCH_CACHE_KEY = 'philo_sobol_search_results'
+DEFAULT_RESULT_TEMPLATE_STRING = "{% if url %}<a href='{{ url }}'>{% endif %}{{ title }}{% if url %}</a>{% endif %}"
+DEFAULT_RESULT_TEMPLATE = Template(DEFAULT_RESULT_TEMPLATE_STRING)
+
+# Determines the timeout on the entire result cache.
+MAX_CACHE_TIMEOUT = 60*24*7
+
+
+class RegistrationError(Exception):
+ pass
+
+
+class SearchRegistry(object):
+ # Holds a registry of search types by slug.
+ def __init__(self):
+ self._registry = {}
+
+ def register(self, search, slug=None):
+ slug = slug or search.slug
+ if slug in self._registry:
+ registered = self._registry[slug]
+ if registered.__module__ != search.__module__:
+ raise RegistrationError("A different search is already registered as `%s`" % slug)
+ else:
+ self._registry[slug] = search
+
+ def unregister(self, search, slug=None):
+ if slug is not None:
+ if slug in self._registry and self._registry[slug] == search:
+ del self._registry[slug]
+ raise RegistrationError("`%s` is not registered as `%s`" % (search, slug))
+ else:
+ for slug, search in self._registry.items():
+ if search == search:
+ del self._registry[slug]
+
+ def items(self):
+ return self._registry.items()
+
+ def iteritems(self):
+ return self._registry.iteritems()
+
+ def iterchoices(self):
+ for slug, search in self.iteritems():
+ yield slug, search.verbose_name
+
+ def __getitem__(self, key):
+ return self._registry[key]
+
+ def __iter__(self):
+ return self._registry.__iter__()
+
+
+registry = SearchRegistry()
+
+
+class Result(object):
+ """
+ A result is instantiated with a configuration dictionary, a search,
+ and a template name. The configuration dictionary is expected to
+ define a `title` and optionally a `url`. Any other variables may be
+ defined; they will be made available through the result object in
+ the template, if one is defined.
+ """
+ def __init__(self, search, result):
+ self.search = search
+ self.result = result
+
+ def get_title(self):
+ return self.search.get_result_title(self.result)
+
+ def get_url(self):
+ qd = self.search.get_result_querydict(self.result)
+ if qd is None:
+ return ""
+ return "?%s" % qd.urlencode()
+
+ def get_template(self):
+ return self.search.get_result_template(self.result)
+
+ def get_extra_context(self):
+ return self.search.get_result_extra_context(self.result)
+
+ def get_context(self):
+ context = self.get_extra_context()
+ context.update({
+ 'title': self.get_title(),
+ 'url': self.get_url()
+ })
+ return context
+
+ def render(self):
+ t = self.get_template()
+ c = Context(self.get_context())
+ return t.render(c)
+
+ def __unicode__(self):
+ return self.render()
+
+
+class BaseSearchMetaclass(type):
+ def __new__(cls, name, bases, attrs):
+ if 'verbose_name' not in attrs:
+ attrs['verbose_name'] = capfirst(convert_camelcase(name))
+ if 'slug' not in attrs:
+ attrs['slug'] = name.lower()
+ return super(BaseSearchMetaclass, cls).__new__(cls, name, bases, attrs)
+
+
+class BaseSearch(object):
+ """
+ Defines a generic search interface. Accessing self.results will
+ attempt to retrieve cached results and, if that fails, will
+ initiate a new search and store the results in the cache.
+ """
+ __metaclass__ = BaseSearchMetaclass
+ result_limit = 10
+ _cache_timeout = 60*48
+
+ def __init__(self, search_arg):
+ self.search_arg = search_arg
+
+ def _get_cached_results(self):
+ """Return the cached results if the results haven't timed out. Otherwise return None."""
+ result_cache = cache.get(SEARCH_CACHE_KEY)
+ if result_cache and self.__class__ in result_cache and self.search_arg.lower() in result_cache[self.__class__]:
+ cached = result_cache[self.__class__][self.search_arg.lower()]
+ if cached['timeout'] >= datetime.datetime.now():
+ return cached['results']
+ return None
+
+ def _set_cached_results(self, results, timeout):
+ """Sets the results to the cache for <timeout> minutes."""
+ result_cache = cache.get(SEARCH_CACHE_KEY) or {}
+ cached = result_cache.setdefault(self.__class__, {}).setdefault(self.search_arg.lower(), {})
+ cached.update({
+ 'results': results,
+ 'timeout': datetime.datetime.now() + datetime.timedelta(minutes=timeout)
+ })
+ cache.set(SEARCH_CACHE_KEY, result_cache, MAX_CACHE_TIMEOUT)
+
+ @property
+ def results(self):
+ if not hasattr(self, '_results'):
+ results = self._get_cached_results()
+ if results is None:
+ try:
+ # Cache one extra result so we can see if there are
+ # more results to be had.
+ limit = self.result_limit
+ if limit is not None:
+ limit += 1
+ results = self.get_results(limit)
+ except:
+ if settings.DEBUG:
+ raise
+ # On exceptions, don't set any cache; just return.
+ return []
+
+ self._set_cached_results(results, self._cache_timeout)
+ self._results = results
+
+ return self._results
+
+ def get_results(self, limit=None, result_class=Result):
+ """
+ Calls self.search() and parses the return value into Result objects.
+ """
+ results = self.search(limit)
+ return [result_class(self, result) for result in results]
+
+ def search(self, limit=None):
+ """
+ Returns an iterable of up to <limit> results. The
+ get_result_title, get_result_url, get_result_template, and
+ get_result_extra_context methods will be used to interpret the
+ individual items that this function returns, so the result can
+ be an object with attributes as easily as a dictionary
+ with keys. The only restriction is that the objects be
+ pickleable so that they can be used with django's cache system.
+ """
+ raise NotImplementedError
+
+ def get_result_title(self, result):
+ raise NotImplementedError
+
+ def get_result_url(self, result):
+ "Subclasses override this to provide the actual URL for the result."
+ raise NotImplementedError
+
+ def get_result_querydict(self, result):
+ url = self.get_result_url(result)
+ if url is None:
+ return None
+ return make_tracking_querydict(self.search_arg, url)
+
+ def get_result_template(self, result):
+ if hasattr(self, 'result_template'):
+ return loader.get_template(self.result_template)
+ if not hasattr(self, '_result_template'):
+ self._result_template = DEFAULT_RESULT_TEMPLATE
+ return self._result_template
+
+ def get_result_extra_context(self, result):
+ return {}
+
+ def has_more_results(self):
+ """Useful to determine whether to display a `view more results` link."""
+ return len(self.results) > self.result_limit
+
+ @property
+ def more_results_url(self):
+ """
+ Returns the actual url for more results. This will be encoded
+ into a querystring for tracking purposes.
+ """
+ raise NotImplementedError
+
+ @property
+ def more_results_querydict(self):
+ return make_tracking_querydict(self.search_arg, self.more_results_url)
+
+ def __unicode__(self):
+ return ' '.join(self.__class__.verbose_name.rsplit(' ', 1)[:-1]) + ' results'
+
+
+class DatabaseSearch(BaseSearch):
+ model = None
+
+ def search(self, limit=None):
+ if not hasattr(self, '_qs'):
+ self._qs = self.get_queryset()
+ if limit is not None:
+ self._qs = self._qs[:limit]
+
+ return self._qs
+
+ def get_queryset(self):
+ return self.model._default_manager.all()
+
+
+class URLSearch(BaseSearch):
+ """
+ Defines a generic interface for searches that require accessing a
+ certain url to get search results.
+ """
+ search_url = ''
+ query_format_str = "%s"
+
+ @property
+ def url(self):
+ "The URL where the search gets its results."
+ return self.search_url + self.query_format_str % urlquote_plus(self.search_arg)
+
+ @property
+ def more_results_url(self):
+ "The URL where the users would go to get more results."
+ return self.url
+
+ def parse_response(self, response, limit=None):
+ raise NotImplementedError
+
+ def search(self, limit=None):
+ return self.parse_response(urllib2.urlopen(self.url), limit=limit)
+
+
+class JSONSearch(URLSearch):
+ """
+ Makes a GET request and parses the results as JSON. The default
+ behavior assumes that the return value is a list of results.
+ """
+ def parse_response(self, response, limit=None):
+ return json.loads(response.read())[:limit]
+
+
+class GoogleSearch(JSONSearch):
+ search_url = "http://ajax.googleapis.com/ajax/services/search/web"
+ # TODO: Change this template to reflect the app's actual name.
+ result_template = 'search/googlesearch.html'
+ _cache_timeout = 60
+ verbose_name = "Google search (current site)"
+
+ @property
+ def query_format_str(self):
+ default_args = self.default_args
+ if default_args:
+ default_args += " "
+ return "?v=1.0&q=%s%%s" % urlquote_plus(default_args).replace('%', '%%')
+
+ @property
+ def default_args(self):
+ return "site:%s" % Site.objects.get_current().domain
+
+ def parse_response(self, response, limit=None):
+ responseData = json.loads(response.read())['responseData']
+ results, cursor = responseData['results'], responseData['cursor']
+
+ if results:
+ self._more_results_url = cursor['moreResultsUrl']
+ self._estimated_result_count = cursor['estimatedResultCount']
+
+ return results[:limit]
+
+ @property
+ def url(self):
+ # Google requires that an ajax request have a proper Referer header.
+ return urllib2.Request(
+ super(GoogleSearch, self).url,
+ None,
+ {'Referer': "http://%s" % Site.objects.get_current().domain}
+ )
+
+ @property
+ def has_more_results(self):
+ if self.results and len(self.results) < self._estimated_result_count:
+ return True
+ return False
+
+ @property
+ def more_results_url(self):
+ return self._more_results_url
+
+ def get_result_title(self, result):
+ return result['titleNoFormatting']
+
+ def get_result_url(self, result):
+ return result['unescapedUrl']
+
+ def get_result_extra_context(self, result):
+ return result
+
+
+registry.register(GoogleSearch)
+
+
+try:
+ from BeautifulSoup import BeautifulSoup, SoupStrainer, BeautifulStoneSoup
+except:
+ pass
+else:
+ __all__ += ('ScrapeSearch', 'XMLSearch',)
+ class ScrapeSearch(URLSearch):
+ _strainer_args = []
+ _strainer_kwargs = {}
+
+ @property
+ def strainer(self):
+ if not hasattr(self, '_strainer'):
+ self._strainer = SoupStrainer(*self._strainer_args, **self._strainer_kwargs)
+ return self._strainer
+
+ def parse_response(self, response, limit=None):
+ strainer = self.strainer
+ soup = BeautifulSoup(response, parseOnlyThese=strainer)
+ return self.parse_results(soup.findAll(recursive=False, limit=limit))
+
+ def parse_results(self, results):
+ """
+ Provides a hook for parsing the results of straining. This
+ has no default behavior because the results absolutely
+ must be parsed to properly extract the information.
+ For more information, see http://www.crummy.com/software/BeautifulSoup/documentation.html#Improving%20Memory%20Usage%20with%20extract
+ """
+ raise NotImplementedError
+
+
+ class XMLSearch(ScrapeSearch):
+ _self_closing_tags = []
+
+ def parse_response(self, response, limit=None):
+ strainer = self.strainer
+ soup = BeautifulStoneSoup(response, selfClosingTags=self._self_closing_tags, parseOnlyThese=strainer)
+ return self.parse_results(soup.findAll(recursive=False, limit=limit))
\ No newline at end of file
git.ithinksw.org / philo.git / blobdiff