auf_savoirs_en_partage_django/savoirs/lib/recherche.py

   1 # -*- encoding: utf-8 -*-
   2 import urllib, httplib, time, simplejson, pprint, math, re
   3 from django.conf import settings
   4 from auf_savoirs_en_partage_backend.sep.io import SEP
   5 from auf_savoirs_en_partage_backend.sep.utils import smart_str
   6 from savoirs import configuration
   7
   8
   9 def google_search (page, q, data):
  10     params = {'q': q,
  11               'rsz': 'large',
  12               'v': '1.0',
  13               'start': page * configuration['resultats_par_page'],
  14               }
  15     if not settings.DEBUG:
  16         #TODO: corriger ts
  17         params['cref'] = "http://savoirsenpartage.auf.org/recherche.xml?%s" \
  18                 % int(time.time())
  19
  20     url = "/ajax/services/search/web?" + \
  21             urllib.urlencode (params)
  22     handle = httplib.HTTPConnection ('ajax.googleapis.com')
  23     handle.request ("GET", url)
  24     r = handle.getresponse ()
  25     response = simplejson.loads(r.read ())
  26     #print pprint.pformat (params)
  27     #print pprint.pformat (response)
  28     handle.close ()
  29
  30     if len (response['responseData']['results']) > 0:
  31         for i in response['responseData']['cursor']['pages']:
  32             p = int (i['label']) - 1
  33             if p > data['last_page']:
  34                 data['last_page'] = p
  35
  36         for r in response['responseData']['results']:
  37             data['results'].append( {'uri': r['url'],
  38                         'content': r['content'],
  39                         'title': r['title']} )
  40
  41         data['more_link'] = response['responseData']['cursor']['moreResultsUrl']
  42
  43
  44 def sep_build_content (regexp, description):
  45     maxlen = 200
  46     content = description
  47     if len (description) > maxlen:
  48         start = 0
  49         loc = regexp.search (description)
  50         if loc:
  51             start = loc.start ()
  52
  53         f = start - (maxlen / 2)
  54         t = 0
  55         if f < 0:
  56             t = -f
  57             f = 0
  58         t += start + (maxlen / 2)
  59         if f > 0:
  60             while description[f] != '.' and f > 0:
  61                 f -= 1
  62             if f > 0:
  63                 f += 1
  64         if t < len (description):
  65             while t < len (description) and description[t] != '.':
  66                 t += 1
  67             t += 1
  68         content = description[f:t]
  69         if f > 0:
  70             content = "(...) " + content
  71         if t < (len (description) - 1):
  72             content = content + " (...)"
  73     content = regexp.sub (r'<b>\1</b>', content)
  74     return content
  75
  76
  77 def make_regexp (q):
  78     words = []
  79     w = re.compile (r'\W+', re.U)
  80     for k in q.keys ():
  81         if k != 'operator':
  82             words.extend(w.split (smart_str(q[k]).decode("utf-8")))
  83     words = filter (lambda x: len(x)>2, words)
  84     words.sort (lambda x,y: len(y)-len(x))
  85
  86     patt = "|".join (words)
  87     patt = "([\W]{1})(" + patt + ")([\W]{1})"
  88     return re.compile (patt, re.I|re.U)
  89
  90 def hl (r, string):
  91     if string is not None:
  92         return r.sub (r'\1<b>\2</b>\3', string)
  93     return None
  94
  95
  96 def sep_search (page, q, data):
  97     f = page * configuration['resultats_par_page']
  98     t = f + 8
  99     s = SEP ()
 100     matches = s.search (q)
 101     data['last_page'] = math.ceil (float(len (matches)) / \
 102             float(configuration['resultats_par_page'])) - 1
 103     set = s.get (matches[f:t])
 104     regexp = make_regexp (q)
 105
 106     for r in set:
 107         uri = r.get ("source", "")
 108         if len (uri) == 0:
 109             uri = r.get ("uri")
 110
 111         title = regexp.sub (r'<b>\1</b>', r.get ("title", ""))
 112
 113         content = sep_build_content (regexp, r.get ("description", ""))
 114
 115         contributeurs = r.get('contributor')
 116         if contributeurs is not None:
 117             contributeurs = "; ".join (contributeurs)
 118
 119         subject = r.get ('subject')
 120         if subject is not None:
 121             subject = ", ".join (subject)
 122
 123         data['results'].append ({'uri': uri,
 124                 'id': r.get("uri"), \
 125                 'title': hl(regexp, title),
 126                 'content': hl(regexp, content), \
 127                 'creator': hl(regexp, r.get('creator')),
 128                 'contributors': hl(regexp, contributeurs),
 129                 'subject': hl(regexp, subject),
 130                 'modified': r.get('modified'),
 131                 'isbn': r.get('isbn'),
 132                 })
 133
 134
 135 def cherche (page, q, engin=None):
 136     rc = {'results': [], 'last_page': 0, 'more_link': ''}
 137
 138     if engin is None:
 139         engin = configuration['engin_recherche']
 140
 141     if engin == 'google':
 142         google_search (page, q, rc)
 143
 144     elif engin == 'sep':
 145         sep_search (page, {'q': q.encode ('utf-8')}, rc)
 146
 147     elif engin == 'avancee':
 148         sep_search (page, q, rc)
 149
 150     return rc