--- /dev/null
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import urllib2
+
+SIGMA_BASE = 'https://formulaires.auf.org/sigma'
+SIGMA_USER = 'XXX'
+SIGMA_PASS = 'XXX'
+
+passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
+passman.add_password(None, SIGMA_BASE, SIGMA_USER, SIGMA_PASS)
+authhandler = urllib2.HTTPBasicAuthHandler(passman)
+opener = urllib2.build_opener(authhandler)
+urllib2.install_opener(opener)
+
+def retrieve(url, filename):
+ doc = urllib2.urlopen(url)
+ f = file(filename, 'wb')
+ f.write(doc.read())
+ f.close()
+ doc.close()
+
+# on récupère quelques pages…
+
+formname = 'demande-de-bourse-de-doctorat-2010-2011-algerie-maroc-tunisie'
+
+for filename in ['field-names.json', 'last-run.log']:
+ url = '%s/%s/%s' % (SIGMA_BASE, formname, filename)
+ retrieve(url, filename) # sauve le contenu à l'URL donnée dans filename
+
+++ /dev/null
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import urllib2
-
-SIGMA_BASE = 'https://formulaires.auf.org/sigma'
-SIGMA_USER = 'XXX'
-SIGMA_PASS = 'XXX'
-
-passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
-passman.add_password(None, SIGMA_BASE, SIGMA_USER, SIGMA_PASS)
-authhandler = urllib2.HTTPBasicAuthHandler(passman)
-opener = urllib2.build_opener(authhandler)
-urllib2.install_opener(opener)
-
-def retrieve(url, filename):
- doc = urllib2.urlopen(url)
- f = file(filename, 'wb')
- f.write(doc.read())
- f.close()
- doc.close()
-
-# on récupère quelques pages…
-
-formname = 'demande-de-bourse-de-doctorat-2010-2011-algerie-maroc-tunisie'
-
-for filename in ['field-names.json', 'last-run.log']:
- url = '%s/%s/%s' % (SIGMA_BASE, formname, filename)
- retrieve(url, filename) # sauve le contenu à l'URL donnée dans filename
-
--- /dev/null
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+import sys
+import twill
+
+if len(sys.argv) != 8:
+ print "Syntax: %s <logname> <logpass> <courriel> <nom> <prenom> <username> <password>" % sys.argv[0]
+ sys.exit(1)
+
+dico = {
+ 'url_login': 'https://formulaires.auf.org/login/',
+ 'username': sys.argv[1],
+ 'password': sys.argv[2],
+ 'url_new_user': 'https://formulaires.auf.org/admin/users/new',
+ 'user_courriel': sys.argv[3],
+ 'user_nom': sys.argv[4],
+ 'user_prenom': sys.argv[5],
+ 'user_username': sys.argv[6],
+ 'user_password': sys.argv[7],
+}
+
+twill.execute_string('''
+go %(url_login)s
+fv 1 username %(username)s
+fv 1 password %(password)s
+submit
+
+go %(url_new_user)s
+fv 1 f0 %(user_courriel)s
+fv 1 f1 '%(user_nom)s'
+fv 1 f2 '%(user_prenom)s'
+fv 1 is_admin False
+fv 1 method_password$username %(user_username)s
+fv 1 method_password$password %(user_password)s
+fv 1 method_password$awaiting_confirmation False
+fv 1 method_password$awaiting_moderation False
+fv 1 method_password$disabled False
+submit submit
+''' % dico)
+
+++ /dev/null
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
-import sys
-import twill
-
-if len(sys.argv) != 8:
- print "Syntax: %s <logname> <logpass> <courriel> <nom> <prenom> <username> <password>" % sys.argv[0]
- sys.exit(1)
-
-dico = {
- 'url_login': 'https://formulaires.auf.org/login/',
- 'username': sys.argv[1],
- 'password': sys.argv[2],
- 'url_new_user': 'https://formulaires.auf.org/admin/users/new',
- 'user_courriel': sys.argv[3],
- 'user_nom': sys.argv[4],
- 'user_prenom': sys.argv[5],
- 'user_username': sys.argv[6],
- 'user_password': sys.argv[7],
-}
-
-twill.execute_string('''
-go %(url_login)s
-fv 1 username %(username)s
-fv 1 password %(password)s
-submit
-
-go %(url_new_user)s
-fv 1 f0 %(user_courriel)s
-fv 1 f1 '%(user_nom)s'
-fv 1 f2 '%(user_prenom)s'
-fv 1 is_admin False
-fv 1 method_password$username %(user_username)s
-fv 1 method_password$password %(user_password)s
-fv 1 method_password$awaiting_confirmation False
-fv 1 method_password$awaiting_moderation False
-fv 1 method_password$disabled False
-submit submit
-''' % dico)
-
--- /dev/null
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import os.path
+import shutil
+import logging
+from time import gmtime, strftime
+import simplejson as json
+
+from wcs import publisher
+from wcs.formdef import FormDef
+from wcs.fields import TitleField, CommentField, TextField, \
+ StringField, ItemField, EmailField, \
+ DateField, FileField, BoolField
+
+
+def reduce_to_alnum(s, replacement_char='-'):
+ """réduction d'une chaîne de caractères à de l'alpha-numérique"""
+
+ avec_accent = u'çÇáàâÁÀÂéèêëÉÈÊËíìîïÍÌÎÏóòôöÓÒÔÖúùûüÚÙÛÜýỳyÿÝỲYŸ'
+ sans_accent = u'cCaaaAAAeeeeEEEEiiiiIIIIooooOOOOuuuuUUUUyyyyYYYY'
+ if type(s) is not unicode:
+ s = unicode(s, 'utf-8')
+ u = False
+ r = ''
+ for c in s:
+ index = avec_accent.find(c)
+ if index >= 0:
+ r += sans_accent[index]
+ elif ('a' <= c.lower() <= 'z') or ('0' <= c <= '9'):
+ r += c
+ elif len(r) > 0 and r[-1] != replacement_char:
+ r += replacement_char
+ else: # r == '' or r[-1] == replacement_char
+ pass
+ r = r.strip(replacement_char)
+ if not u:
+ r = r.encode('utf-8')
+ return r
+
+
+def extract_fields(formdef, output_directory):
+ """nommage des champs de façon unique"""
+ # TODO: devrait retourner un résultat, qui serait alors sauvé en dehors
+
+ # XXX: hack temporaire… :-/
+ global field_names
+
+ f = open(os.path.join(output_directory, 'field-names.txt'), 'w')
+
+ field_names = {}
+ field_names_duplicates = {}
+ for field in formdef.fields:
+ if isinstance(field, TitleField) or isinstance(field, CommentField):
+ continue
+ name = reduce_to_alnum(field.label,'_').lower()
+ if name in field_names.values(): # duplicat
+ field_names_duplicates[name] = field_names_duplicates.get(name, 1) + 1
+ name = '%s_%d' % (name, field_names_duplicates[name])
+ field_names.update({field.id: name})
+ print >>f, "%s:%s:%s" % (field.id, field_names[field.id], field.label)
+
+ f.close()
+
+ f = open(os.path.join(output_directory, 'field-names.json'), 'wb')
+ f.write(json.dumps(field_names, ensure_ascii=False))
+ f.close()
+
+
+def extract_data(formdef, output_directory):
+ """extraction des données du formulaire"""
+ # TODO: devrait retourner un résultat, qui serait alors sauvé en dehors
+
+ # XXX: hack temporaire… :-/
+ global pub
+
+ liste_dossiers = []
+ for object in formdef.data_class().select():
+ result = {
+ 'num_dossier': object.id,
+ 'wcs_status': object.status,
+ 'wcs_workflow_status': object.get_workflow_status().name,
+ 'wcs_user_email': object.user.email,
+ 'wcs_user_display_name': object.user.display_name,
+ #'wcs_last_modified': strftime('%Y-%m-%d %H:%M:%S', gmtime(object.last_modified())),
+ 'wcs_comments': [],
+ }
+
+ if object.evolution is not None:
+ for e in object.evolution:
+ if e.comment is not None:
+ who = pub.user_class.get(e.who).display_name
+ time = strftime('%Y-%m-%d %H:%M:%S', e.time)
+ comment = '%s -- %s %s' % (e.comment, who, time)
+ result['wcs_comments'].append(comment)
+
+ qfiles = { }
+ for field in formdef.fields:
+ field_id = str(field.id)
+ if not field_id in object.data:
+ continue
+ if isinstance(field, TitleField) or isinstance(field, CommentField):
+ continue
+ field_name = field_names[field_id]
+ data = object.data.get(field_id)
+ if isinstance(field, StringField) or isinstance(field, TextField) \
+ or isinstance(field, EmailField) or isinstance(field, ItemField):
+ result[field_name] = data
+ elif isinstance(field, BoolField):
+ result[field_name] = (data == 'True')
+ elif isinstance(field, DateField):
+ result[field_name] = strftime('%Y-%m-%d', data)
+ elif isinstance(field, FileField):
+ extension = data.orig_filename.rpartition('.')[2].lower()
+ result[field_name] = "%s.%s" % (field_name, extension)
+ qfiles[field_name] = data.qfilename
+ else:
+ logging.error("Type de champ inconnu '%s' pour '%s'.",
+ field.__class__.__name__, field.label)
+ raise RuntimeError
+
+ num_dossier = result['num_dossier']
+ nom = reduce_to_alnum(result['nom']).upper()
+ prenom = reduce_to_alnum(result['prenom']).upper()
+ adel = result['adresse_electronique'].replace('@','-').lower()
+
+ filename = "%04d-%s-%s-%s" % (num_dossier, nom, prenom, adel)
+ liste_dossiers.append(filename + '.json')
+
+ # copie des fichiers joints
+ for f in qfiles:
+ result[f] = filename + '_' + result[f]
+ src = os.path.join(pub.app_dir, 'uploads', qfiles[f])
+ dst = os.path.join(output_directory, 'data', result[f])
+ if not os.path.exists(dst) or os.path.getmtime(src) > os.path.getmtime(dst):
+ shutil.copy2(src, dst)
+ os.chmod(dst, 0644)
+
+ # génération du fichier JSON
+ jsonname = os.path.join(output_directory, 'data', filename + '.json')
+ f = open(jsonname, 'wb')
+ f.write(json.dumps(result, ensure_ascii=False))
+ f.close()
+
+ logging.info("Dossier '%s' : %s.",
+ filename, result['wcs_workflow_status'])
+
+ f = open(os.path.join(output_directory, 'liste-dossiers.json'), 'wb')
+ f.write(json.dumps(liste_dossiers, ensure_ascii=False))
+ f.close()
+
+
+if __name__ == '__main__':
+ import sys
+
+ if len(sys.argv) != 4:
+ print >>sys.stderr, "Usage : %s <dossier-destination> <site> <formulaire>" % sys.argv[0]
+ sys.exit(1)
+
+ VHOST = sys.argv[2]
+ FORM_NAME = sys.argv[3]
+ OUTPUT_DIRECTORY = os.path.join(sys.argv[1], VHOST, FORM_NAME)
+
+ os.umask(0022)
+ # création du dossier d'extraction, au besoin
+ if not os.path.isdir(OUTPUT_DIRECTORY):
+ os.makedirs(OUTPUT_DIRECTORY, 0755)
+
+ logging.basicConfig(level=logging.DEBUG,
+ format='%(asctime)s %(levelname)s %(message)s',
+ filename=os.path.join(OUTPUT_DIRECTORY, 'last-run.log'),
+ filemode='w')
+
+ logging.info('Début.')
+
+ pub = publisher.WcsPublisher.create_publisher()
+ pub.app_dir = os.path.join(pub.app_dir, VHOST)
+
+ formdef = FormDef.get_by_urlname(FORM_NAME)
+
+ extract_fields(formdef, OUTPUT_DIRECTORY)
+
+ extract_data(formdef, OUTPUT_DIRECTORY)
+
+ logging.info('Fin.')
+
--- /dev/null
+#!/bin/sh
+OUTPUT_DIR="/srv/wcs"
+SITE="formulaires.auf.org"
+FORMS="\
+ horizons-francophones-2010-bureau-ocean-indien \
+ demande-de-bourse-de-doctorat-2010-2011-algerie-maroc-tunisie \
+ demande-de-bourse-de-doctorat-2011-2012-afrique-centrale-et-des-grands-lacs \
+"
+
+# XXX: hack en attendant de corriger le umask de wcs…
+sudo chmod -R g+rX /var/lib/wcs/
+
+echo "Traitement des formulaires de '$SITE'…"
+for f in $FORMS
+do
+ echo " $f"
+ python wcs-extract.py $OUTPUT_DIR $SITE $f
+done
+echo "Terminé."
+++ /dev/null
-#!/bin/sh
-OUTPUT_DIR="/srv/wcs"
-SITE="formulaires.auf.org"
-FORMS="\
- horizons-francophones-2010-bureau-ocean-indien \
- demande-de-bourse-de-doctorat-2010-2011-algerie-maroc-tunisie \
- demande-de-bourse-de-doctorat-2011-2012-afrique-centrale-et-des-grands-lacs \
-"
-
-# XXX: hack en attendant de corriger le umask de wcs…
-sudo chmod -R g+rX /var/lib/wcs/
-
-echo "Traitement des formulaires de '$SITE'…"
-for f in $FORMS
-do
- echo " $f"
- python wcs-extract.py $OUTPUT_DIR $SITE $f
-done
-echo "Terminé."
+++ /dev/null
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import os
-import os.path
-import shutil
-import logging
-from time import gmtime, strftime
-import simplejson as json
-
-from wcs import publisher
-from wcs.formdef import FormDef
-from wcs.fields import TitleField, CommentField, TextField, \
- StringField, ItemField, EmailField, \
- DateField, FileField, BoolField
-
-
-def reduce_to_alnum(s, replacement_char='-'):
- """réduction d'une chaîne de caractères à de l'alpha-numérique"""
-
- avec_accent = u'çÇáàâÁÀÂéèêëÉÈÊËíìîïÍÌÎÏóòôöÓÒÔÖúùûüÚÙÛÜýỳyÿÝỲYŸ'
- sans_accent = u'cCaaaAAAeeeeEEEEiiiiIIIIooooOOOOuuuuUUUUyyyyYYYY'
- if type(s) is not unicode:
- s = unicode(s, 'utf-8')
- u = False
- r = ''
- for c in s:
- index = avec_accent.find(c)
- if index >= 0:
- r += sans_accent[index]
- elif ('a' <= c.lower() <= 'z') or ('0' <= c <= '9'):
- r += c
- elif len(r) > 0 and r[-1] != replacement_char:
- r += replacement_char
- else: # r == '' or r[-1] == replacement_char
- pass
- r = r.strip(replacement_char)
- if not u:
- r = r.encode('utf-8')
- return r
-
-
-def extract_fields(formdef, output_directory):
- """nommage des champs de façon unique"""
- # TODO: devrait retourner un résultat, qui serait alors sauvé en dehors
-
- # XXX: hack temporaire… :-/
- global field_names
-
- f = open(os.path.join(output_directory, 'field-names.txt'), 'w')
-
- field_names = {}
- field_names_duplicates = {}
- for field in formdef.fields:
- if isinstance(field, TitleField) or isinstance(field, CommentField):
- continue
- name = reduce_to_alnum(field.label,'_').lower()
- if name in field_names.values(): # duplicat
- field_names_duplicates[name] = field_names_duplicates.get(name, 1) + 1
- name = '%s_%d' % (name, field_names_duplicates[name])
- field_names.update({field.id: name})
- print >>f, "%s:%s:%s" % (field.id, field_names[field.id], field.label)
-
- f.close()
-
- f = open(os.path.join(output_directory, 'field-names.json'), 'wb')
- f.write(json.dumps(field_names, ensure_ascii=False))
- f.close()
-
-
-def extract_data(formdef, output_directory):
- """extraction des données du formulaire"""
- # TODO: devrait retourner un résultat, qui serait alors sauvé en dehors
-
- liste_dossiers = []
- for object in formdef.data_class().select():
- result = {
- 'num_dossier': object.id,
- 'wcs_status': object.status,
- 'wcs_workflow_status': object.get_workflow_status().name,
- 'wcs_user_email': object.user.email,
- 'wcs_user_display_name': object.user.display_name,
- #'wcs_last_modified': strftime('%Y-%m-%d %H:%M:%S', gmtime(object.last_modified())),
- }
- qfiles = { }
- for field in formdef.fields:
- field_id = str(field.id)
- if not field_id in object.data:
- continue
- if isinstance(field, TitleField) or isinstance(field, CommentField):
- continue
- field_name = field_names[field_id]
- data = object.data.get(field_id)
- if isinstance(field, StringField) or isinstance(field, TextField) \
- or isinstance(field, EmailField) or isinstance(field, ItemField):
- result[field_name] = data
- elif isinstance(field, BoolField):
- result[field_name] = (data == 'True')
- elif isinstance(field, DateField):
- result[field_name] = strftime('%Y-%m-%d', data)
- elif isinstance(field, FileField):
- extension = data.orig_filename.rpartition('.')[2].lower()
- result[field_name] = "%s.%s" % (field_name, extension)
- qfiles[field_name] = data.qfilename
- else:
- logging.error("Type de champ inconnu '%s' pour '%s'.",
- field.__class__.__name__, field.label)
- raise RuntimeError
-
- num_dossier = result['num_dossier']
- nom = reduce_to_alnum(result['nom']).upper()
- prenom = reduce_to_alnum(result['prenom']).upper()
- adel = result['adresse_electronique'].replace('@','-').lower()
-
- filename = "%04d-%s-%s-%s" % (num_dossier, nom, prenom, adel)
- liste_dossiers.append(filename + '.json')
-
- # copie des fichiers joints
- for f in qfiles:
- result[f] = filename + '_' + result[f]
- src = os.path.join(pub.app_dir, 'uploads', qfiles[f])
- dst = os.path.join(output_directory, 'data', result[f])
- if not os.path.exists(dst) or os.path.getmtime(src) > os.path.getmtime(dst):
- shutil.copy2(src, dst)
- os.chmod(dst, 0644)
-
- # génération du fichier JSON
- jsonname = os.path.join(output_directory, 'data', filename + '.json')
- f = open(jsonname, 'wb')
- f.write(json.dumps(result, ensure_ascii=False))
- f.close()
-
- logging.info("Dossier '%s' : %s.",
- filename, result['wcs_workflow_status'])
-
- f = open(os.path.join(output_directory, 'liste-dossiers.json'), 'wb')
- f.write(json.dumps(liste_dossiers, ensure_ascii=False))
- f.close()
-
-
-if __name__ == '__main__':
- import sys
-
- if len(sys.argv) != 4:
- print >>sys.stderr, "Usage : %s <dossier-destination> <site> <formulaire>" % sys.argv[0]
- sys.exit(1)
-
- VHOST = sys.argv[2]
- FORM_NAME = sys.argv[3]
- OUTPUT_DIRECTORY = os.path.join(sys.argv[1], VHOST, FORM_NAME)
-
- os.umask(0022)
- # création du dossier d'extraction, au besoin
- if not os.path.isdir(OUTPUT_DIRECTORY):
- os.makedirs(OUTPUT_DIRECTORY, 0755)
-
- logging.basicConfig(level=logging.DEBUG,
- format='%(asctime)s %(levelname)s %(message)s',
- filename=os.path.join(OUTPUT_DIRECTORY, 'last-run.log'),
- filemode='w')
-
- logging.info('Début.')
-
- pub = publisher.WcsPublisher.create_publisher()
- pub.app_dir = os.path.join(pub.app_dir, VHOST)
-
- formdef = FormDef.get_by_urlname(FORM_NAME)
-
- extract_fields(formdef, OUTPUT_DIRECTORY)
-
- extract_data(formdef, OUTPUT_DIRECTORY)
-
- logging.info('Fin.')
-
--- /dev/null
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+
+if len(sys.argv) != 3:
+ print >>sys.stderr, "Usage : %s <site> <formulaire>" % sys.argv[0]
+ sys.exit(1)
+VHOST = sys.argv[1]
+FORM_NAME = sys.argv[2]
+
+PRINT_FORMAT = "%4s | %-17s | %-52s"
+PRINT_HEADERS = PRINT_FORMAT % ('Num.', 'Statut', 'Utilisateur')
+PRINT_BAR = "=" * len(PRINT_HEADERS)
+
+from wcs import publisher
+from wcs.formdef import FormDef
+
+pub = publisher.WcsPublisher.create_publisher()
+pub.app_dir = os.path.join(pub.app_dir, VHOST)
+
+formdef = FormDef.get_by_urlname(FORM_NAME)
+
+print PRINT_HEADERS
+print PRINT_BAR
+
+for object in formdef.data_class().select():
+ print PRINT_FORMAT % ('%04d' % object.id,
+ object.get_workflow_status().name,
+ '%s <%s>' % (object.user.display_name, object.user.email),
+ )
+
+print PRINT_BAR
+++ /dev/null
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import os
-import sys
-
-if len(sys.argv) != 3:
- print >>sys.stderr, "Usage : %s <site> <formulaire>" % sys.argv[0]
- sys.exit(1)
-VHOST = sys.argv[1]
-FORM_NAME = sys.argv[2]
-
-PRINT_FORMAT = "%4s | %-17s | %-52s"
-PRINT_HEADERS = PRINT_FORMAT % ('Num.', 'Statut', 'Utilisateur')
-PRINT_BAR = "=" * len(PRINT_HEADERS)
-
-from wcs import publisher
-from wcs.formdef import FormDef
-
-pub = publisher.WcsPublisher.create_publisher()
-pub.app_dir = os.path.join(pub.app_dir, VHOST)
-
-formdef = FormDef.get_by_urlname(FORM_NAME)
-
-print PRINT_HEADERS
-print PRINT_BAR
-
-for object in formdef.data_class().select():
- print PRINT_FORMAT % ('%04d' % object.id,
- object.get_workflow_status().name,
- '%s <%s>' % (object.user.display_name, object.user.email),
- )
-
-print PRINT_BAR
projets / progfou.git / commitdiff
