auf_savoirs_en_partage/savoirs/lib/harvest.py

   1 # -*- encoding: utf-8 -*-
   2 import sys, os, time, traceback
   3 from auf_savoirs_en_partage.backend_config import RESOURCES
   4 from savoirs.models import HarvestLog
   5 from sep import SEP
   6
   7 def import_all ():
   8     """Cette méthode effectue l'importation des données pour toutes les
   9     sources définies dans `conf.py`, et les ajoute dans le système de stockage
  10     en passant par SEP (:doc:`../sep/index`)
  11     """
  12     sep = SEP ()
  13
  14     resources = RESOURCES
  15     if len(sys.argv) == 2:
  16         name = sys.argv[1]
  17         if RESOURCES.get(name) is not None:
  18             resources = {name: RESOURCES.get(name)}
  19         else:
  20             print "Ressource %s non existante" % name
  21             sys.exit(-1)
  22
  23     for name in resources.keys ():
  24         print "Import:", name
  25         options = RESOURCES[name]
  26         options['server'] = name
  27
  28         module = 'harvesters.%s.%s' \
  29                 % (options['type'], options['acces'])
  30         __import__ (module)
  31         harvester = sys.modules[module]
  32         try:
  33             nodes = harvester.harvest (options)
  34         except:
  35             print "Exception:"
  36             print '-'*60
  37             traceback.print_exc(file=sys.stdout)
  38             print '-'*60
  39             nodes = []
  40
  41         added = updated = 0
  42         for node in nodes:
  43             node['server'] = name
  44
  45             try:
  46                 status = sep.add (node)
  47             except:
  48                 message.update({'context':'error', 'name':name, 'processed':0})
  49                 HarvestLog.add(message)
  50                 continue
  51
  52             if status['added']:
  53                 added += 1
  54             if status['updated']:
  55                 updated += 1
  56             message = status
  57             message.update({'context':'record', 'name':name, 'processed':1})
  58             HarvestLog.add(message)
  59
  60         message = {'context':'moisson', 'name':name, 'added':added, 'updated':updated, 'processed':len(nodes)}
  61         HarvestLog.add(message)
  62
  63     del (sep)
  64
  65 if __name__ == '__main__':
  66     import_all()