auf_savoirs_en_partage/savoirs/lib/harvest.py

   1 # -*- encoding: utf-8 -*-
   2 import sys, os, time, traceback
   3
   4 from auf_savoirs_en_partage.backend_config import RESOURCES
   5 from sep import SEP
   6
   7
   8 def import_all ():
   9     """Cette méthode effectue l'importation des données pour toutes les
  10     sources définies dans `conf.py`, et les ajoute dans le système de stockage
  11     en passant par SEP (:doc:`../sep/index`)
  12     """
  13     sep = SEP ()
  14
  15     resources = RESOURCES
  16     if len(sys.argv) == 2:
  17         name = sys.argv[1]
  18         if RESOURCES.get(name) is not None:
  19             resources = {name: RESOURCES.get(name)}
  20         else:
  21             print "Ressource %s non existante" % name
  22             sys.exit(-1)
  23
  24     for name in resources.keys ():
  25         print "Import:", name
  26         options = RESOURCES[name]
  27         module = 'harvesters.%s.%s' \
  28                 % (options['type'], options['acces'])
  29         __import__ (module)
  30         harvester = sys.modules[module]
  31         try:
  32             nodes = harvester.harvest (options)
  33         except:
  34             print "Exception:"
  35             print '-'*60
  36             traceback.print_exc(file=sys.stdout)
  37             print '-'*60
  38             nodes = []
  39         print "Ajout de", len(nodes), "references"
  40         print "S:", time.time ()
  41         for node in nodes:
  42             sep.add (node)
  43         sep.add_log (name, len(nodes))
  44         print "F:", time.time ()
  45
  46     del (sep)
  47
  48 if __name__ == '__main__':
  49     import_all()