2 # -*- coding: utf-8 -*-
4 Outil d'export de données w.c.s.
6 Copyright : Agence universitaire de la Francophonie — www.auf.org
7 Licence : GNU General Public Licence, version 2
8 Auteur : Jean Christophe André
9 Date de création : 15 octobre 2009
11 Depends: wcs, python-simplejson, python-magic
17 from time import gmtime, strftime
18 import simplejson as json
22 from wcs import publisher
23 from wcs.formdef import FormDef
24 from wcs.fields import TitleField, CommentField, TextField, \
25 StringField, ItemField, EmailField, \
26 DateField, FileField, BoolField
29 def reduce_to_alnum(s, replacement_char='-'):
30 """réduction d'une chaîne de caractères à de l'alpha-numérique"""
32 avec_accent = u'çÇáàâÁÀÂéèêëÉÈÊËíìîïÍÌÎÏóòôöÓÒÔÖúùûüÚÙÛÜýỳyÿÝỲYŸ'
33 sans_accent = u'cCaaaAAAeeeeEEEEiiiiIIIIooooOOOOuuuuUUUUyyyyYYYY'
34 if type(s) is not unicode:
35 s = unicode(s, 'utf-8')
39 index = avec_accent.find(c)
41 r += sans_accent[index]
42 elif ('a' <= c.lower() <= 'z') or ('0' <= c <= '9'):
44 elif len(r) > 0 and r[-1] != replacement_char:
46 else: # r == '' or r[-1] == replacement_char
48 r = r.strip(replacement_char)
54 def extract_fields(formdef, output_directory):
55 """nommage des champs de façon unique"""
56 # TODO: devrait retourner un résultat, qui serait alors sauvé en dehors
58 # XXX: hack temporaire… :-/
61 f = open(os.path.join(output_directory, 'field-names.txt'), 'w')
64 field_names_duplicates = {}
65 for field in formdef.fields:
66 if isinstance(field, TitleField) or isinstance(field, CommentField):
68 name = reduce_to_alnum(field.label,'_').lower()
69 if name in field_names.values(): # duplicat
70 field_names_duplicates[name] = field_names_duplicates.get(name, 1) + 1
71 name = '%s_%d' % (name, field_names_duplicates[name])
72 field_names.update({field.id: name})
73 print >>f, "%s:%s:%s" % (field.id, field_names[field.id], field.label)
77 f = open(os.path.join(output_directory, 'field-names.json'), 'wb')
78 f.write(json.dumps(field_names, ensure_ascii=False, sort_keys=True))
82 def extract_data(formdef, output_directory):
83 """extraction des données du formulaire"""
84 # TODO: devrait retourner un résultat, qui serait alors sauvé en dehors
86 # XXX: hack temporaire… :-/
89 # on charge la base des types MIME une fois pour toutes
90 #magicmime = magic.Magic(mime=True) => ce sera pour plus tard…
91 magicmime = magic.open(magic.MAGIC_MIME)
94 for object in formdef.data_class().select():
96 'num_dossier': object.id,
97 'wcs_status': object.status,
98 'wcs_workflow_status': object.get_workflow_status().name,
99 'wcs_user_email': object.user.email,
100 'wcs_user_display_name': object.user.display_name,
101 #'wcs_last_modified': strftime('%Y-%m-%d %H:%M:%S', gmtime(object.last_modified())),
105 if object.evolution is not None:
106 for e in object.evolution:
107 if e.comment is not None:
108 who = pub.user_class.get(e.who).display_name
109 time = strftime('%Y-%m-%d %H:%M:%S', e.time)
110 comment = '%s -- %s %s' % (e.comment, who, time)
111 result['wcs_comments'].append(comment)
114 for field in formdef.fields:
115 field_id = str(field.id)
116 if not field_id in object.data:
118 if isinstance(field, TitleField) or isinstance(field, CommentField):
120 field_name = field_names[field_id]
121 data = object.data.get(field_id)
122 if isinstance(field, StringField) or isinstance(field, TextField) \
123 or isinstance(field, EmailField) or isinstance(field, ItemField):
124 result[field_name] = data
125 elif isinstance(field, BoolField):
126 result[field_name] = (data == 'True')
127 elif isinstance(field, DateField):
128 result[field_name] = strftime('%Y-%m-%d', data)
129 elif isinstance(field, FileField):
130 if '.' in data.orig_filename:
131 extension = data.orig_filename.rpartition('.')[2].lower()
132 else: # il n'y a pas d'extension dans le nom de fichier
133 p = os.path.join(pub.app_dir, 'uploads', data.qfilename)
134 #m = magicmime.from_file(p) => ce sera pour plus tard…
135 m = magicmime.file(p).split()[0].strip(';')
136 extension = mimetypes.guess_extension(m)
137 if extension is not None:
138 extension = extension[1:]
140 extension = 'unknown'
141 result[field_name] = "%s.%s" % (field_name, extension)
142 qfiles[field_name] = data.qfilename
144 logging.error("Type de champ inconnu '%s' pour '%s'.",
145 field.__class__.__name__, field.label)
148 num_dossier = result['num_dossier']
149 nom = reduce_to_alnum(result['nom']).upper()
150 prenom = reduce_to_alnum(result['prenom']).upper()
151 adel = result['adresse_electronique'].replace('@','-').lower()
153 filename = "%04d-%s-%s-%s" % (num_dossier, nom, prenom, adel)
154 liste_dossiers.append(filename + '.json')
156 # copie des fichiers joints
158 result[f] = filename + '_' + result[f]
159 src = os.path.join(pub.app_dir, 'uploads', qfiles[f])
160 dst = os.path.join(output_directory, 'data', result[f])
161 if not os.path.exists(dst) or os.path.getmtime(src) > os.path.getmtime(dst):
162 shutil.copy2(src, dst)
165 # génération du fichier JSON
166 jsonname = os.path.join(output_directory, 'data', filename + '.json')
167 f = open(jsonname, 'wb')
168 f.write(json.dumps(result, ensure_ascii=False, sort_keys=True))
171 logging.info("Dossier '%s' : %s.",
172 filename, result['wcs_workflow_status'])
174 f = open(os.path.join(output_directory, 'liste-dossiers.json'), 'wb')
175 f.write(json.dumps(liste_dossiers, ensure_ascii=False, sort_keys=True))
179 if __name__ == '__main__':
182 if len(sys.argv) != 4:
183 print >>sys.stderr, "Usage : %s <dossier-destination> <site> <formulaire>" % sys.argv[0]
187 FORM_NAME = sys.argv[3]
188 OUTPUT_DIRECTORY = os.path.join(sys.argv[1], VHOST, FORM_NAME)
191 # création du dossier d'extraction, au besoin
192 if not os.path.isdir(OUTPUT_DIRECTORY):
193 os.makedirs(OUTPUT_DIRECTORY, 0755)
195 logging.basicConfig(level=logging.DEBUG,
196 format='%(asctime)s %(levelname)s %(message)s',
197 filename=os.path.join(OUTPUT_DIRECTORY, 'last-run.log'),
200 logging.info('Début.')
202 pub = publisher.WcsPublisher.create_publisher()
203 pub.app_dir = os.path.join(pub.app_dir, VHOST)
205 formdef = FormDef.get_by_urlname(FORM_NAME)
207 extract_fields(formdef, OUTPUT_DIRECTORY)
209 extract_data(formdef, OUTPUT_DIRECTORY)