[progfou.git] / wcs / wcs-extract

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Outil d'export de données w.c.s.

Copyright : Agence universitaire de la Francophonie — www.auf.org
Licence : GNU General Public Licence, version 2
Auteur : Jean Christophe André
Date de création : 15 octobre 2009

Depends: wcs, python-simplejson, python-magic
"""
import os
import os.path
import shutil
import logging
from time import gmtime, strftime
import simplejson as json
import magic
import mimetypes

from wcs import publisher
from wcs.formdef import FormDef
from wcs.fields import TitleField, CommentField, TextField, \
                       StringField, ItemField, EmailField, \
                       DateField, FileField, BoolField


def reduce_to_alnum(s, replacement_char='-'):
    """réduction d'une chaîne de caractères à de l'alpha-numérique"""

    avec_accent = u'çÇáàâÁÀÂéèêëÉÈÊËíìîïÍÌÎÏóòôöÓÒÔÖúùûüÚÙÛÜýỳyÿÝỲYŸ'
    sans_accent = u'cCaaaAAAeeeeEEEEiiiiIIIIooooOOOOuuuuUUUUyyyyYYYY'
    if type(s) is not unicode:
        s = unicode(s, 'utf-8')
        u  = False
    r = ''
    for c in s:
        index = avec_accent.find(c)
        if index >= 0:
            r += sans_accent[index]
        elif ('a' <= c.lower() <= 'z') or ('0' <= c <= '9'):
            r += c
        elif len(r) > 0 and r[-1] != replacement_char:
            r += replacement_char
        else: # r == '' or r[-1] == replacement_char
            pass
    r = r.strip(replacement_char)
    if not u:
        r = r.encode('utf-8')
    return r


def extract_fields(formdef, output_directory):
    """nommage des champs de façon unique"""
    # TODO: devrait retourner un résultat, qui serait alors sauvé en dehors

    # XXX: hack temporaire… :-/
    global field_names

    f = open(os.path.join(output_directory, 'field-names.txt'), 'w')

    field_names = {}
    field_names_duplicates = {}
    for field in formdef.fields:
        if isinstance(field, TitleField) or isinstance(field, CommentField):
            continue
        name = reduce_to_alnum(field.label,'_').lower()
        if name in field_names.values(): # duplicat
            field_names_duplicates[name] = field_names_duplicates.get(name, 1) + 1
            name = '%s_%d' % (name, field_names_duplicates[name])
        field_names.update({field.id: name})
        print >>f, "%s:%s:%s" % (field.id, field_names[field.id], field.label)

    f.close()

    f = open(os.path.join(output_directory, 'field-names.json'), 'wb')
    f.write(json.dumps(field_names, ensure_ascii=False))
    f.close()


def extract_data(formdef, output_directory):
    """extraction des données du formulaire"""
    # TODO: devrait retourner un résultat, qui serait alors sauvé en dehors

    # XXX: hack temporaire… :-/
    global pub

    # on charge la base des types MIME une fois pour toutes
    #magicmime = magic.Magic(mime=True) => ce sera pour plus tard…
    magicmime = magic.open(magic.MAGIC_MIME)
    magicmime.load()

    liste_dossiers = []
    for object in formdef.data_class().select():
        if object.user is None:
            logging.warning("Dossier '%s' sans utilisateur associé ?!?"\
                            " On ignore...", object.id)
            continue

        result = {
            'num_dossier': object.id,
            'wcs_status': object.status,
            'wcs_workflow_status': object.get_workflow_status().name,
            'wcs_user_email': object.user.email,
            'wcs_user_display_name': object.user.display_name,
           #'wcs_last_modified': strftime('%Y-%m-%d %H:%M:%S', gmtime(object.last_modified())),
            'wcs_comments': [],
        }

        if object.evolution is not None:
            for e in object.evolution:
                if e.comment is not None:
                    who = pub.user_class.get(e.who).display_name
                    time = strftime('%Y-%m-%d %H:%M:%S', e.time)
                    comment = '%s -- %s %s' % (e.comment, who, time)
                    result['wcs_comments'].append(comment)

        qfiles = { }
        for field in formdef.fields:
            field_id = str(field.id)
            if not field_id in object.data:
                continue
            if isinstance(field, TitleField) or isinstance(field, CommentField):
                continue
            field_name = field_names[field_id]
            data = object.data.get(field_id)
            if isinstance(field, StringField) or isinstance(field, TextField) \
            or isinstance(field, EmailField) or isinstance(field, ItemField):
                result[field_name] = data
            elif isinstance(field, BoolField):
                result[field_name] = (data == 'True')
            elif isinstance(field, DateField):
                result[field_name] = '%04d-%02d-%02d' % (data.tm_year,
                                                    data.tm_mon, data.tm_mday)
            elif isinstance(field, FileField) and data is not None:
                if '.' in data.orig_filename:
                    extension = data.orig_filename.rpartition('.')[2].lower()
                else: # il n'y a pas d'extension dans le nom de fichier
                    p = os.path.join(pub.app_dir, 'uploads', data.qfilename)
                    try:
                        #m = magicmime.from_file(p) => ce sera pour plus tard…
                        m = magicmime.file(p).split()[0].strip(';')
                        extension = mimetypes.guess_extension(m)
                    except:
                        logging.warning("Type de fichier inconnu pour '%s'.", p)
                        extension = None
                    if extension is not None:
                        extension = extension[1:]
                    else:
                        extension = 'unknown'
                result[field_name] = "%s.%s" % (field_name, extension)
                qfiles[field_name] = data.qfilename
            else:
                logging.error("Type de champ inconnu '%s' pour '%s'.",
                                    field.__class__.__name__, field.label)
                raise RuntimeError

        num_dossier = result['num_dossier']
        nom = reduce_to_alnum(result['nom']).upper()
        prenom = reduce_to_alnum(result['prenom']).upper()
        adel = result['adresse_electronique'].replace('@','-').lower()

        filename = "%04d-%s-%s-%s" % (num_dossier, nom, prenom, adel)
        liste_dossiers.append(filename + '.json')

        # copie des fichiers joints
        for f in qfiles:
            result[f] = filename + '_' + result[f]
            src = os.path.join(pub.app_dir, 'uploads', qfiles[f])
            dst = os.path.join(output_directory, 'data', result[f])
            if not os.path.exists(dst) or os.path.getmtime(src) > os.path.getmtime(dst):
                shutil.copy2(src, dst)
                os.chmod(dst, 0644)

        # génération du fichier JSON
        jsonname = os.path.join(output_directory, 'data', filename + '.json')
        f = open(jsonname, 'wb')
        f.write(json.dumps(result, ensure_ascii=False))
        f.close()

        logging.info("Dossier '%s' : %s.",
                                    filename, result['wcs_workflow_status'])

    liste_dossiers.sort()
    f = open(os.path.join(output_directory, 'liste-dossiers.json'), 'wb')
    f.write(json.dumps(liste_dossiers, ensure_ascii=False))
    f.close()


if __name__ == '__main__':
    import sys

    if len(sys.argv) != 4:
        print >>sys.stderr, "Usage : %s <dossier-destination> <site> <formulaire>" % sys.argv[0]
        sys.exit(1)

    VHOST = sys.argv[2]
    FORM_NAME = sys.argv[3]
    OUTPUT_DIRECTORY = os.path.join(sys.argv[1], VHOST, FORM_NAME)

    os.umask(0022)
    # création du dossier d'extraction, au besoin
    if not os.path.isdir(os.path.join(OUTPUT_DIRECTORY, 'data')):
        os.makedirs(os.path.join(OUTPUT_DIRECTORY, 'data'), 0755)

    logging.basicConfig(level=logging.DEBUG,
        format='%(asctime)s %(levelname)s %(message)s',
        filename=os.path.join(OUTPUT_DIRECTORY, 'last-run.log'),
        filemode='w')

    logging.info('Début.')

    pub = publisher.WcsPublisher.create_publisher()
    pub.app_dir = os.path.join(pub.app_dir, VHOST)

    formdef = FormDef.get_by_urlname(FORM_NAME)

    extract_fields(formdef, OUTPUT_DIRECTORY)

    try:
        extract_data(formdef, OUTPUT_DIRECTORY)
    except:
        logging.exception("Interruption du traitement pour cause d'erreur !")

    logging.info('Fin.')
Commit	Line	Data
0f48356a P	1	#!/usr/bin/env python
0f48356a P	2	# -- coding: utf-8 --
edf3eba0 P	3	"""
edf3eba0 P	4	Outil d'export de données w.c.s.
0f48356a	5
edf3eba0 P	6	Copyright : Agence universitaire de la Francophonie — www.auf.org
	7	Licence : GNU General Public Licence, version 2
	8	Auteur : Jean Christophe André
	9	Date de création : 15 octobre 2009
	10
	11	Depends: wcs, python-simplejson, python-magic
	12	"""
0f48356a P	13	import os
0f48356a P	14	import os.path
a9de62a1 P	15	import shutil
a9de62a1 P	16	import logging
0f48356a P	17	from time import gmtime, strftime
0f48356a P	18	import simplejson as json
edf3eba0 P	19	import magic
edf3eba0 P	20	import mimetypes
0f48356a	21
0cd61ae9 P	22	from wcs import publisher
	23	from wcs.formdef import FormDef
	24	from wcs.fields import TitleField, CommentField, TextField, \
	25	StringField, ItemField, EmailField, \
	26	DateField, FileField, BoolField
0f48356a	27
0f48356a	28
0cd61ae9 P	29	def reduce_to_alnum(s, replacement_char='-'):
	30	"""réduction d'une chaîne de caractères à de l'alpha-numérique"""
	31
0f48356a P	32	avec_accent = u'çÇáàâÁÀÂéèêëÉÈÊËíìîïÍÌÎÏóòôöÓÒÔÖúùûüÚÙÛÜýỳyÿÝỲYŸ'
	33	sans_accent = u'cCaaaAAAeeeeEEEEiiiiIIIIooooOOOOuuuuUUUUyyyyYYYY'
	34	if type(s) is not unicode:
	35	s = unicode(s, 'utf-8')
	36	u = False
	37	r = ''
	38	for c in s:
	39	index = avec_accent.find(c)
	40	if index >= 0:
	41	r += sans_accent[index]
	42	elif ('a' <= c.lower() <= 'z') or ('0' <= c <= '9'):
	43	r += c
	44	elif len(r) > 0 and r[-1] != replacement_char:
	45	r += replacement_char
	46	else: # r == '' or r[-1] == replacement_char
	47	pass
	48	r = r.strip(replacement_char)
	49	if not u:
	50	r = r.encode('utf-8')
	51	return r
	52
0f48356a	53
0cd61ae9 P	54	def extract_fields(formdef, output_directory):
	55	"""nommage des champs de façon unique"""
	56	# TODO: devrait retourner un résultat, qui serait alors sauvé en dehors
	57
36319452 P	58	# XXX: hack temporaire… :-/
	59	global field_names
	60
0cd61ae9 P	61	f = open(os.path.join(output_directory, 'field-names.txt'), 'w')
	62
	63	field_names = {}
	64	field_names_duplicates = {}
0f48356a	65	for field in formdef.fields:
0f48356a P	66	if isinstance(field, TitleField) or isinstance(field, CommentField):
0f48356a P	67	continue
0cd61ae9 P	68	name = reduce_to_alnum(field.label,'_').lower()
	69	if name in field_names.values(): # duplicat
	70	field_names_duplicates[name] = field_names_duplicates.get(name, 1) + 1
	71	name = '%s_%d' % (name, field_names_duplicates[name])
	72	field_names.update({field.id: name})
	73	print >>f, "%s:%s:%s" % (field.id, field_names[field.id], field.label)
	74
0f48356a P	75	f.close()
0f48356a P	76
0cd61ae9	77	f = open(os.path.join(output_directory, 'field-names.json'), 'wb')
47d9d835	78	f.write(json.dumps(field_names, ensure_ascii=False))
0cd61ae9 P	79	f.close()
	80
	81
	82	def extract_data(formdef, output_directory):
	83	"""extraction des données du formulaire"""
	84	# TODO: devrait retourner un résultat, qui serait alors sauvé en dehors
	85
e80345a7 P	86	# XXX: hack temporaire… :-/
	87	global pub
	88
edf3eba0 P	89	# on charge la base des types MIME une fois pour toutes
edf3eba0 P	90	#magicmime = magic.Magic(mime=True) => ce sera pour plus tard…
6244020f	91	magicmime = magic.open(magic.MAGIC_MIME)
dffcfa61	92	magicmime.load()
edf3eba0	93
2e14236b	94	liste_dossiers = []
0cd61ae9	95	for object in formdef.data_class().select():
9d424de0 P	96	if object.user is None:
	97	logging.warning("Dossier '%s' sans utilisateur associé ?!?"\
	98	" On ignore...", object.id)
	99	continue
	100
0cd61ae9 P	101	result = {
	102	'num_dossier': object.id,
	103	'wcs_status': object.status,
	104	'wcs_workflow_status': object.get_workflow_status().name,
	105	'wcs_user_email': object.user.email,
	106	'wcs_user_display_name': object.user.display_name,
	107	#'wcs_last_modified': strftime('%Y-%m-%d %H:%M:%S', gmtime(object.last_modified())),
e80345a7	108	'wcs_comments': [],
0cd61ae9	109	}
e80345a7 P	110
	111	if object.evolution is not None:
	112	for e in object.evolution:
	113	if e.comment is not None:
	114	who = pub.user_class.get(e.who).display_name
	115	time = strftime('%Y-%m-%d %H:%M:%S', e.time)
	116	comment = '%s -- %s %s' % (e.comment, who, time)
	117	result['wcs_comments'].append(comment)
	118
0cd61ae9 P	119	qfiles = { }
	120	for field in formdef.fields:
	121	field_id = str(field.id)
	122	if not field_id in object.data:
	123	continue
	124	if isinstance(field, TitleField) or isinstance(field, CommentField):
	125	continue
	126	field_name = field_names[field_id]
	127	data = object.data.get(field_id)
	128	if isinstance(field, StringField) or isinstance(field, TextField) \
	129	or isinstance(field, EmailField) or isinstance(field, ItemField):
	130	result[field_name] = data
	131	elif isinstance(field, BoolField):
	132	result[field_name] = (data == 'True')
	133	elif isinstance(field, DateField):
b182779a P	134	result[field_name] = '%04d-%02d-%02d' % (data.tm_year,
	135	data.tm_mon, data.tm_mday)
	136	elif isinstance(field, FileField) and data is not None:
edf3eba0 P	137	if '.' in data.orig_filename:
	138	extension = data.orig_filename.rpartition('.')[2].lower()
	139	else: # il n'y a pas d'extension dans le nom de fichier
	140	p = os.path.join(pub.app_dir, 'uploads', data.qfilename)
474e5135 P	141	try:
	142	#m = magicmime.from_file(p) => ce sera pour plus tard…
	143	m = magicmime.file(p).split()[0].strip(';')
	144	extension = mimetypes.guess_extension(m)
	145	except:
	146	logging.warning("Type de fichier inconnu pour '%s'.", p)
	147	extension = None
edf3eba0 P	148	if extension is not None:
	149	extension = extension[1:]
	150	else:
	151	extension = 'unknown'
0cd61ae9 P	152	result[field_name] = "%s.%s" % (field_name, extension)
	153	qfiles[field_name] = data.qfilename
	154	else:
a9de62a1 P	155	logging.error("Type de champ inconnu '%s' pour '%s'.",
a9de62a1 P	156	field.__class__.__name__, field.label)
0cd61ae9 P	157	raise RuntimeError
	158
	159	num_dossier = result['num_dossier']
	160	nom = reduce_to_alnum(result['nom']).upper()
	161	prenom = reduce_to_alnum(result['prenom']).upper()
	162	adel = result['adresse_electronique'].replace('@','-').lower()
	163
	164	filename = "%04d-%s-%s-%s" % (num_dossier, nom, prenom, adel)
2e14236b	165	liste_dossiers.append(filename + '.json')
0cd61ae9	166
0cd61ae9 P	167	# copie des fichiers joints
	168	for f in qfiles:
	169	result[f] = filename + '_' + result[f]
	170	src = os.path.join(pub.app_dir, 'uploads', qfiles[f])
a9de62a1	171	dst = os.path.join(output_directory, 'data', result[f])
0cd61ae9	172	if not os.path.exists(dst) or os.path.getmtime(src) > os.path.getmtime(dst):
a9de62a1	173	shutil.copy2(src, dst)
0cd61ae9 P	174	os.chmod(dst, 0644)
	175
	176	# génération du fichier JSON
a9de62a1 P	177	jsonname = os.path.join(output_directory, 'data', filename + '.json')
a9de62a1 P	178	f = open(jsonname, 'wb')
47d9d835	179	f.write(json.dumps(result, ensure_ascii=False))
0cd61ae9 P	180	f.close()
0cd61ae9 P	181
a9de62a1 P	182	logging.info("Dossier '%s' : %s.",
a9de62a1 P	183	filename, result['wcs_workflow_status'])
0cd61ae9	184
b35749bb	185	liste_dossiers.sort()
2e14236b	186	f = open(os.path.join(output_directory, 'liste-dossiers.json'), 'wb')
b35749bb	187	f.write(json.dumps(liste_dossiers, ensure_ascii=False))
2e14236b P	188	f.close()
2e14236b P	189
0cd61ae9 P	190
	191	if __name__ == '__main__':
	192	import sys
	193
	194	if len(sys.argv) != 4:
	195	print >>sys.stderr, "Usage : %s <dossier-destination> <site> <formulaire>" % sys.argv[0]
	196	sys.exit(1)
	197
	198	VHOST = sys.argv[2]
	199	FORM_NAME = sys.argv[3]
	200	OUTPUT_DIRECTORY = os.path.join(sys.argv[1], VHOST, FORM_NAME)
	201
	202	os.umask(0022)
	203	# création du dossier d'extraction, au besoin
9d424de0 P	204	if not os.path.isdir(os.path.join(OUTPUT_DIRECTORY, 'data')):
9d424de0 P	205	os.makedirs(os.path.join(OUTPUT_DIRECTORY, 'data'), 0755)
0cd61ae9	206
a9de62a1 P	207	logging.basicConfig(level=logging.DEBUG,
	208	format='%(asctime)s %(levelname)s %(message)s',
	209	filename=os.path.join(OUTPUT_DIRECTORY, 'last-run.log'),
	210	filemode='w')
	211
	212	logging.info('Début.')
	213
0cd61ae9 P	214	pub = publisher.WcsPublisher.create_publisher()
	215	pub.app_dir = os.path.join(pub.app_dir, VHOST)
	216
	217	formdef = FormDef.get_by_urlname(FORM_NAME)
	218
	219	extract_fields(formdef, OUTPUT_DIRECTORY)
	220
b182779a P	221	try:
	222	extract_data(formdef, OUTPUT_DIRECTORY)
	223	except:
	224	logging.exception("Interruption du traitement pour cause d'erreur !")
0cd61ae9	225
a9de62a1	226	logging.info('Fin.')
0f48356a	227