Exemple d'extraction des données de w.c.s, ici pour import dans Sigma.

[progfou.git] / wcs / wcs2json.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

# le nom d'hote hébergeant wcs
VHOST = "formulaires.auf.org"
# nom du formulaire à explorer
FORM_NAME = "demande-de-bourse-de-doctorat-2010-2011"
# nom des fichiers à générer (un par formulaire)
OUTPUT_DIRECTORY = "/srv/www/wcs/%s/%s" % (VHOST, FORM_NAME)

URL_BASE = "http://%s/backoffice/%s" % (VHOST, FORM_NAME)
URL_DOWNLOAD = "%s/%s" % (URL_BASE, "%s/download?f=%s")
#TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
TIME_FORMAT = "%Y-%m-%d"

import os
import sys
from wcs import publisher
from wcs.formdef import FormDef
from wcs.fields import TitleField, CommentField, TextField, \
                       StringField, ItemField, EmailField, \
                       DateField, FileField, BoolField
from time import strftime
from simplejson import dumps, loads

def cleanup(s):
    avec_accent = u'çÇáàâÁÀÂéèêëÉÈÊËíìîïÍÌÎÏóòôöÓÒÔÖúùûüÚÙÛÜýỳyÿÝỲYŸ'
    sans_accent = u'cCaaaAAAeeeeEEEEiiiiIIIIooooOOOOuuuuUUUUyyyyYYYY'
    if type(s) is not unicode:
        s = unicode(s, 'utf-8')
        u  = False
    r = ''
    for c in s:
        index = avec_accent.find(c)
        if index >= 0:
            c = sans_accent[index]
        elif c != '-' and not ('a' <= c.lower() <= 'z'):
            c = '-'
        r += c
    if not u:
        r = r.encode('utf-8')
    return r

f = open(FORM_NAME + '_field-names.txt', 'r')
field_names = dict([l.strip().split(':') for l in f.readlines()])
f.close()

qfile_commands = open(FORM_NAME + '_copy-qfiles.sh', 'w')

pub = publisher.WcsPublisher.create_publisher()
pub.app_dir = os.path.join(pub.app_dir, VHOST)


formdef = FormDef.get_by_urlname(FORM_NAME)
for object in formdef.data_class().select():
    result = { 'num_dossier': object.id }
    qfiles = { }
    for field in formdef.fields:
        field_id = str(field.id)
        if not field_id in object.data:
            continue
        if isinstance(field, TitleField) or isinstance(field, CommentField):
            continue
        field_name = field_names.get(field_id, field.label)
        data = object.data.get(field_id)
        if isinstance(field, StringField) or isinstance(field, TextField) \
        or isinstance(field, EmailField):
            result[field_name] = data
        elif isinstance(field, ItemField):
            if field_name.startswith('pays') or 'etablissement' in field_name:
                index = data.find('(')
                if index >= 0:
                    end = data.find(' ', index+1)
                    data = data[index+1:end]
            elif field_name == 'discipline':
                index = data.rfind('(')
                if index >= 0:
                    end = data.find(')', index+1)
                    data = data[index+1:end]
            #elif field_name == 'region':
            #    index = data.rfind('(')
            #    if index >= 0:
            #        index = data.find('-', index+1)
            #        end = data.find(')', index+1)
            #        data = filter(lambda c: c.isupper(), data[index+1:end])
            result[field_name] = data
        elif isinstance(field, BoolField):
            result[field_name] = (data == 'True')
        elif isinstance(field, FileField):
            extension = data.orig_filename.rpartition('.')[2].lower()
            result[field_name] = "%s.%s" % (field_name, extension)
            qfiles[field_name] = data.qfilename
        elif isinstance(field, DateField):
            data = strftime(TIME_FORMAT, object.data.get(field_id))
            result[field_name] = data
        else:
            data = object.data.get(field_id)
            print "WARNING: unknown field type '%s' for '%s'" % \
                                    (field.__class__.__name__, field.label)
            raise RuntimeError

    num_dossier = result['num_dossier']
    nom = cleanup('-'.join(result['nom'].split()).upper())
    prenom = cleanup('-'.join(result['prenom'].split()).upper())
    email = result['email'].replace('@','-').lower()

    filename = "%s/%04d-%s-%s-%s.json" % \
        (OUTPUT_DIRECTORY, num_dossier, nom, prenom, email)

    f = open(filename, 'wb')
    f.write(dumps(result, ensure_ascii=False).replace(', ',',\n'))
    f.close()

    filename = filename[:-len('.json')]
    for f in qfiles:
        print >>qfile_commands, "cp -af '%s/uploads/%s' '%s_%s'" % \
                        (pub.app_dir, qfiles[f], filename, result[f])

qfile_commands.close()