auf_savoirs_en_partage/savoirs/lib/harvest.py

   1 # -*- encoding: utf-8 -*-
   2 import sys, os, time, traceback
   3 from auf_savoirs_en_partage.backend_config import RESOURCES
   4 from savoirs.models import HarvestLog
   5 from sep import SEP
   6
   7 def import_all ():
   8     """Cette méthode effectue l'importation des données pour toutes les
   9     sources définies dans `conf.py`, et les ajoute dans le système de stockage
  10     en passant par SEP (:doc:`../sep/index`)
  11     """
  12     sep = SEP ()
  13
  14     resources = RESOURCES
  15     if len(sys.argv) == 2:
  16         name = sys.argv[1]
  17         if RESOURCES.get(name) is not None:
  18             resources = {name: RESOURCES.get(name)}
  19         else:
  20             print "Ressource %s non existante" % name
  21             sys.exit(-1)
  22
  23     for name in resources.keys ():
  24         print "Import:", name
  25         options = RESOURCES[name]
  26         options['server'] = name
  27
  28         module = 'harvesters.%s.%s' \
  29                 % (options['type'], options['acces'])
  30         __import__ (module)
  31         harvester = sys.modules[module]
  32         try:
  33             nodes = harvester.harvest (options)
  34         except:
  35             print "Exception:"
  36             print '-'*60
  37             traceback.print_exc(file=sys.stdout)
  38             print '-'*60
  39             nodes = []
  40
  41         added = updated = 0
  42         for node in nodes:
  43             node['server'] = name
  44             status = sep.add (node)
  45
  46             if status['added']:
  47                 added += 1
  48             if status['updated']:
  49                 updated += 1
  50             message = status
  51             message.update({'context':'record', 'name':name})
  52             HarvestLog.add(message)
  53
  54         message = {'context':'moisson', 'name':name, 'added':added, 'updated':updated}
  55         HarvestLog.add(message)
  56
  57     del (sep)
  58
  59 if __name__ == '__main__':
  60     import_all()