Commit | Line | Data |
---|---|---|
0f48356a P |
1 | #!/usr/bin/env python |
2 | # -*- coding: utf-8 -*- | |
edf3eba0 P |
3 | """ |
4 | Outil d'export de données w.c.s. | |
0f48356a | 5 | |
edf3eba0 P |
6 | Copyright : Agence universitaire de la Francophonie — www.auf.org |
7 | Licence : GNU General Public Licence, version 2 | |
8 | Auteur : Jean Christophe André | |
9 | Date de création : 15 octobre 2009 | |
10 | ||
11 | Depends: wcs, python-simplejson, python-magic | |
12 | """ | |
0f48356a P |
13 | import os |
14 | import os.path | |
a9de62a1 P |
15 | import shutil |
16 | import logging | |
0f48356a P |
17 | from time import gmtime, strftime |
18 | import simplejson as json | |
edf3eba0 P |
19 | import magic |
20 | import mimetypes | |
0f48356a | 21 | |
0cd61ae9 P |
22 | from wcs import publisher |
23 | from wcs.formdef import FormDef | |
24 | from wcs.fields import TitleField, CommentField, TextField, \ | |
25 | StringField, ItemField, EmailField, \ | |
26 | DateField, FileField, BoolField | |
0f48356a | 27 | |
0f48356a | 28 | |
0cd61ae9 P |
29 | def reduce_to_alnum(s, replacement_char='-'): |
30 | """réduction d'une chaîne de caractères à de l'alpha-numérique""" | |
31 | ||
0f48356a P |
32 | avec_accent = u'çÇáàâÁÀÂéèêëÉÈÊËíìîïÍÌÎÏóòôöÓÒÔÖúùûüÚÙÛÜýỳyÿÝỲYŸ' |
33 | sans_accent = u'cCaaaAAAeeeeEEEEiiiiIIIIooooOOOOuuuuUUUUyyyyYYYY' | |
34 | if type(s) is not unicode: | |
35 | s = unicode(s, 'utf-8') | |
36 | u = False | |
37 | r = '' | |
38 | for c in s: | |
39 | index = avec_accent.find(c) | |
40 | if index >= 0: | |
41 | r += sans_accent[index] | |
42 | elif ('a' <= c.lower() <= 'z') or ('0' <= c <= '9'): | |
43 | r += c | |
44 | elif len(r) > 0 and r[-1] != replacement_char: | |
45 | r += replacement_char | |
46 | else: # r == '' or r[-1] == replacement_char | |
47 | pass | |
48 | r = r.strip(replacement_char) | |
49 | if not u: | |
50 | r = r.encode('utf-8') | |
51 | return r | |
52 | ||
0f48356a | 53 | |
0cd61ae9 P |
54 | def extract_fields(formdef, output_directory): |
55 | """nommage des champs de façon unique""" | |
56 | # TODO: devrait retourner un résultat, qui serait alors sauvé en dehors | |
57 | ||
36319452 P |
58 | # XXX: hack temporaire… :-/ |
59 | global field_names | |
60 | ||
0cd61ae9 P |
61 | f = open(os.path.join(output_directory, 'field-names.txt'), 'w') |
62 | ||
63 | field_names = {} | |
64 | field_names_duplicates = {} | |
0f48356a | 65 | for field in formdef.fields: |
0f48356a P |
66 | if isinstance(field, TitleField) or isinstance(field, CommentField): |
67 | continue | |
0cd61ae9 P |
68 | name = reduce_to_alnum(field.label,'_').lower() |
69 | if name in field_names.values(): # duplicat | |
70 | field_names_duplicates[name] = field_names_duplicates.get(name, 1) + 1 | |
71 | name = '%s_%d' % (name, field_names_duplicates[name]) | |
72 | field_names.update({field.id: name}) | |
73 | print >>f, "%s:%s:%s" % (field.id, field_names[field.id], field.label) | |
74 | ||
0f48356a P |
75 | f.close() |
76 | ||
0cd61ae9 | 77 | f = open(os.path.join(output_directory, 'field-names.json'), 'wb') |
47d9d835 | 78 | f.write(json.dumps(field_names, ensure_ascii=False)) |
0cd61ae9 P |
79 | f.close() |
80 | ||
81 | ||
82 | def extract_data(formdef, output_directory): | |
83 | """extraction des données du formulaire""" | |
84 | # TODO: devrait retourner un résultat, qui serait alors sauvé en dehors | |
85 | ||
e80345a7 P |
86 | # XXX: hack temporaire… :-/ |
87 | global pub | |
88 | ||
edf3eba0 P |
89 | # on charge la base des types MIME une fois pour toutes |
90 | #magicmime = magic.Magic(mime=True) => ce sera pour plus tard… | |
6244020f | 91 | magicmime = magic.open(magic.MAGIC_MIME) |
dffcfa61 | 92 | magicmime.load() |
edf3eba0 | 93 | |
2e14236b | 94 | liste_dossiers = [] |
0cd61ae9 | 95 | for object in formdef.data_class().select(): |
9d424de0 P |
96 | if object.user is None: |
97 | logging.warning("Dossier '%s' sans utilisateur associé ?!?"\ | |
98 | " On ignore...", object.id) | |
99 | continue | |
100 | ||
0cd61ae9 P |
101 | result = { |
102 | 'num_dossier': object.id, | |
103 | 'wcs_status': object.status, | |
104 | 'wcs_workflow_status': object.get_workflow_status().name, | |
105 | 'wcs_user_email': object.user.email, | |
106 | 'wcs_user_display_name': object.user.display_name, | |
107 | #'wcs_last_modified': strftime('%Y-%m-%d %H:%M:%S', gmtime(object.last_modified())), | |
e80345a7 | 108 | 'wcs_comments': [], |
0cd61ae9 | 109 | } |
e80345a7 P |
110 | |
111 | if object.evolution is not None: | |
112 | for e in object.evolution: | |
113 | if e.comment is not None: | |
114 | who = pub.user_class.get(e.who).display_name | |
115 | time = strftime('%Y-%m-%d %H:%M:%S', e.time) | |
116 | comment = '%s -- %s %s' % (e.comment, who, time) | |
117 | result['wcs_comments'].append(comment) | |
118 | ||
0cd61ae9 P |
119 | qfiles = { } |
120 | for field in formdef.fields: | |
121 | field_id = str(field.id) | |
122 | if not field_id in object.data: | |
123 | continue | |
124 | if isinstance(field, TitleField) or isinstance(field, CommentField): | |
125 | continue | |
126 | field_name = field_names[field_id] | |
127 | data = object.data.get(field_id) | |
128 | if isinstance(field, StringField) or isinstance(field, TextField) \ | |
129 | or isinstance(field, EmailField) or isinstance(field, ItemField): | |
130 | result[field_name] = data | |
131 | elif isinstance(field, BoolField): | |
132 | result[field_name] = (data == 'True') | |
133 | elif isinstance(field, DateField): | |
b182779a P |
134 | result[field_name] = '%04d-%02d-%02d' % (data.tm_year, |
135 | data.tm_mon, data.tm_mday) | |
136 | elif isinstance(field, FileField) and data is not None: | |
edf3eba0 P |
137 | if '.' in data.orig_filename: |
138 | extension = data.orig_filename.rpartition('.')[2].lower() | |
139 | else: # il n'y a pas d'extension dans le nom de fichier | |
140 | p = os.path.join(pub.app_dir, 'uploads', data.qfilename) | |
474e5135 P |
141 | try: |
142 | #m = magicmime.from_file(p) => ce sera pour plus tard… | |
143 | m = magicmime.file(p).split()[0].strip(';') | |
144 | extension = mimetypes.guess_extension(m) | |
145 | except: | |
146 | logging.warning("Type de fichier inconnu pour '%s'.", p) | |
147 | extension = None | |
edf3eba0 P |
148 | if extension is not None: |
149 | extension = extension[1:] | |
150 | else: | |
151 | extension = 'unknown' | |
0cd61ae9 P |
152 | result[field_name] = "%s.%s" % (field_name, extension) |
153 | qfiles[field_name] = data.qfilename | |
154 | else: | |
a9de62a1 P |
155 | logging.error("Type de champ inconnu '%s' pour '%s'.", |
156 | field.__class__.__name__, field.label) | |
0cd61ae9 P |
157 | raise RuntimeError |
158 | ||
159 | num_dossier = result['num_dossier'] | |
160 | nom = reduce_to_alnum(result['nom']).upper() | |
161 | prenom = reduce_to_alnum(result['prenom']).upper() | |
162 | adel = result['adresse_electronique'].replace('@','-').lower() | |
163 | ||
164 | filename = "%04d-%s-%s-%s" % (num_dossier, nom, prenom, adel) | |
2e14236b | 165 | liste_dossiers.append(filename + '.json') |
0cd61ae9 | 166 | |
0cd61ae9 P |
167 | # copie des fichiers joints |
168 | for f in qfiles: | |
169 | result[f] = filename + '_' + result[f] | |
170 | src = os.path.join(pub.app_dir, 'uploads', qfiles[f]) | |
a9de62a1 | 171 | dst = os.path.join(output_directory, 'data', result[f]) |
0cd61ae9 | 172 | if not os.path.exists(dst) or os.path.getmtime(src) > os.path.getmtime(dst): |
a9de62a1 | 173 | shutil.copy2(src, dst) |
0cd61ae9 P |
174 | os.chmod(dst, 0644) |
175 | ||
176 | # génération du fichier JSON | |
a9de62a1 P |
177 | jsonname = os.path.join(output_directory, 'data', filename + '.json') |
178 | f = open(jsonname, 'wb') | |
47d9d835 | 179 | f.write(json.dumps(result, ensure_ascii=False)) |
0cd61ae9 P |
180 | f.close() |
181 | ||
a9de62a1 P |
182 | logging.info("Dossier '%s' : %s.", |
183 | filename, result['wcs_workflow_status']) | |
0cd61ae9 | 184 | |
b35749bb | 185 | liste_dossiers.sort() |
2e14236b | 186 | f = open(os.path.join(output_directory, 'liste-dossiers.json'), 'wb') |
b35749bb | 187 | f.write(json.dumps(liste_dossiers, ensure_ascii=False)) |
2e14236b P |
188 | f.close() |
189 | ||
0cd61ae9 P |
190 | |
191 | if __name__ == '__main__': | |
192 | import sys | |
193 | ||
194 | if len(sys.argv) != 4: | |
195 | print >>sys.stderr, "Usage : %s <dossier-destination> <site> <formulaire>" % sys.argv[0] | |
196 | sys.exit(1) | |
197 | ||
198 | VHOST = sys.argv[2] | |
199 | FORM_NAME = sys.argv[3] | |
200 | OUTPUT_DIRECTORY = os.path.join(sys.argv[1], VHOST, FORM_NAME) | |
201 | ||
202 | os.umask(0022) | |
203 | # création du dossier d'extraction, au besoin | |
9d424de0 P |
204 | if not os.path.isdir(os.path.join(OUTPUT_DIRECTORY, 'data')): |
205 | os.makedirs(os.path.join(OUTPUT_DIRECTORY, 'data'), 0755) | |
0cd61ae9 | 206 | |
a9de62a1 P |
207 | logging.basicConfig(level=logging.DEBUG, |
208 | format='%(asctime)s %(levelname)s %(message)s', | |
209 | filename=os.path.join(OUTPUT_DIRECTORY, 'last-run.log'), | |
210 | filemode='w') | |
211 | ||
212 | logging.info('Début.') | |
213 | ||
0cd61ae9 P |
214 | pub = publisher.WcsPublisher.create_publisher() |
215 | pub.app_dir = os.path.join(pub.app_dir, VHOST) | |
216 | ||
217 | formdef = FormDef.get_by_urlname(FORM_NAME) | |
218 | ||
219 | extract_fields(formdef, OUTPUT_DIRECTORY) | |
220 | ||
b182779a P |
221 | try: |
222 | extract_data(formdef, OUTPUT_DIRECTORY) | |
223 | except: | |
224 | logging.exception("Interruption du traitement pour cause d'erreur !") | |
0cd61ae9 | 225 | |
a9de62a1 | 226 | logging.info('Fin.') |
0f48356a | 227 |