path: root/src/local/domain/cri.rhizi.net/from_csv

#!/usr/bin/python2.7

"""
Import CSV files into Rhizi Server via pythonic API.

Tricky, since this is the only API user. Try to use it exactly as the REST/WS
API would, just without actually creating a socket connection.
"""

import sys
import os
import argparse
import string
import csv

root = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..'))
print("root = %s" % root)
sys.path.append(os.path.join(root, 'src', 'server'))

from rz_api_common import sanitize_input__topo_diff
from rz_kernel import RZ_Kernel
import db_controller as dbc
from rz_server import Config

from model.graph import Topo_Diff

def topo_diff_json(node_set_add=[], link_set_add=[]):
    topo_diff_dict = ({
         u'link_id_set_rm': [],
         u'link_set_add': link_set_add,
         u'drop_conjugator_links': True,
         u'node_set_add': node_set_add,
         u'node_id_set_rm': []
        })
    topo_diff = Topo_Diff.from_json_dict(topo_diff_dict)
    sanitize_input__topo_diff(topo_diff)
    return topo_diff;

def commit(topo_diff):
    _, commit_ret = kernel.diff_commit__topo(topo_diff, ctx)

# Rhizi constants - FIXME use API
PERSON_LABEL = 'Person'
INTEREST_LABEL = 'Interest'
INTERNSHIP_LABEL = "Internship"
LABEL_SET = '__label_set'

# CSV file columns
PERSONAL_EMAIL = 'Personal e-mail'
INTEREST = 'Interests'

id_count = 0
def next_id():
    global id_count
    id_count += 1
    return '%08d' % id_count

def node_dict(the_id, **args):
    """ FIXME use API """
    ret = dict(args)
    ret['id'] = the_id
    return ret

def link_dict(source_id, target_id, label, the_id):
    return ({
               u'__type': [label],
               u'__src_id': source_id,
               u'__dst_id': target_id,
               u'id': the_id
           })

def read_csv(filename):
    for dialect in ['excel', 'excel-tab']:
        lines = list(csv.reader(open(filename), dialect=dialect))
        lens = map(len, lines)
        if max(lens) == min(lens) and max(lens) > 2:
            return lines
    raise Exception("no dialect found")

class CSV(object):
    def __init__(self, filename):
        self.filename = filename
        rows = read_csv(filename)
        headers = rows[0]
        self.parse_headers(headers)
        #import pdb; pdb.set_trace()
        self.rows = rows[1:]
        self.row_dicts = map(lambda fields: dict(zip(headers, fields)), self.rows)

    def run(self):
        print("    ****************************    run on data from %r     ****************" % self.filename)
        self.nodes_dict = {}
        self.node_set_add = []
        self.link_set_add = []
        self.generate_nodes_and_links()
        #for node in node_set_add:
        #    print('committing %s' % repr(node))
        #    commit(topo_diff_json(node_set_add=[node]))
        commit(topo_diff_json(node_set_add=self.node_set_add))
        commit(topo_diff_json(link_set_add=self.link_set_add))

    def append_id_node(self, the_id, node):
        if the_id in self.nodes_dict:
            return
        print(repr(node))
        self.nodes_dict[the_id] = node
        self.node_set_add.append(node)
        return the_id

    def _csv_row_to_node(self, d, dict_gen):
        the_id = next_id()
        print(repr(d))
        return the_id, node_dict(the_id=the_id, **dict_gen(d))

    def append_link(self, source_id, target_id, label):
        the_id = next_id()
        link = link_dict(source_id, target_id, label, the_id)
        self.link_set_add.append(link)
        print(repr(link))
        return the_id

class Persons(object):
    def __init__(self):
        self.by_name = {}
        self.by_email = {}

    def id_from_something(self, first_name, last_name, email):
        return self.by_name.get(self.name_from_first_and_last(first_name, last_name), self.by_email.get(email, (None, None)))[0]

    def name_from_first_and_last(self, first_name, last_name):
        return first_name + ' ' + last_name

    def addPersonNode(self, first_name, last_name, email):
        ret = ret2 = None
        name = self.name_from_first_and_last(first_name, last_name)
        if name in self.by_name:
            ret = self.by_name[name]
        if email in self.by_email:
            ret2 = self.by_email[email]
        if ret != ret2:
            print("person node similar but not same for: %r, %r, %r" % (first_name, last_name, email))
            print(ret2)
        if ret2 and not ret:
            ret = ret2
        if ret:
            return ret
        node_id = next_id()
        node = node_dict(the_id=node_id, name=name, description=email, **{LABEL_SET: [PERSON_LABEL]})
        ret = (node_id, node)
        self.by_name[name] = ret
        self.by_email[email] = ret
        return ret

persons = Persons()
addPersonNode = persons.addPersonNode

def cleanEmailField(unclean):
    return unclean.lower().strip()

class StudentCSV(CSV):

    def __init__(self, filename, **kw):
        super(StudentCSV, self).__init__(filename=filename, **kw)

    def interests(self, d):
        return d[self.interestsField]

    def generate_nodes_and_links(self):
        for d in self.row_dicts:
            d[self.interestsField] = map(string.strip, d[self.interestsField].split(','))
            person_id = next_id()
            person_email = cleanEmailField(d[self.personalEmailField])
            person_id, person_node = addPersonNode(first_name=d[self.firstNameField], last_name=d[self.lastNameField], email=person_email)
            self.append_id_node(person_id, person_node)
            for interest in self.interests(d):
                interest_id = next_id()
                interest_node = node_dict(the_id=interest_id,
                    **{'name':interest, LABEL_SET:[INTEREST_LABEL]})
                interest_id = self.append_id_node(interest_id, interest_node)
                self.append_link(person_id, interest_id, 'Is interested in')

    def parse_headers(self, headers):
        self.firstNameField = headers[0]
        self.lastNameField = headers[1]
        self.personalEmailField = headers[2]
        self.interestsField = headers[3]

descriptionTemplate = """Nature: %s
--------
Abstract:
%s"""

class StudentInternshipsCSV(CSV):

    def __init__(self, filename, **kw):
        super(StudentInternshipsCSV, self).__init__(filename=filename, **kw)

    def parse_headers(self, headers):
        it = iter(headers)
        self.firstNameField = it.next()
        self.lastNameField = it.next()
        self.emailField = it.next()
        self.internshipTitleField = it.next()
        self.internshipNatureField = it.next()
        self.abstractField = it.next()
        self.dateStartField = it.next()
        self.dateEndField = it.next()
        self.supervisorFirstNameField = it.next()
        self.supervisorLastNameField = it.next()
        self.supervisorTitleField = it.next()
        self.supervisorEmailField = it.next()
        self.laboratoryNameField = it.next()
        self.laboratoryAffiliationField = it.next()
        self.unitCodeField = it.next() # CNRS  / INSERM unit code
        self.streetField = it.next() # Street
        self.streetSecondPartField = it.next() # Street (continued)
        self.cityField = it.next() # City
        self.countryField = it.next() # Country

    def generate_nodes_and_links(self):
        for d in self.row_dicts:
            internship_id = next_id()
            # FIXME should be title. fix requires client change. need
            # configurable fields, or a list of names and types for display per
            # node (using a type node in the db, which could be cached
            title = d[self.internshipTitleField].strip()
            if len(title) == 0:
                print("skipping empty internship field")
                continue
            input_dict = {
                         LABEL_SET:[INTERNSHIP_LABEL],
                         'name': title,
                         'description': descriptionTemplate % (d[self.internshipNatureField], d[self.abstractField]),
                         'startdate': d[self.dateStartField],
                         'enddate': d[self.dateEndField],
                         'internship-type': d[self.internshipNatureField],
                         'cnrs-inserm-unit-code': d[self.unitCodeField],
                         'city': d[self.cityField],
                         'country': d[self.countryField],
                         'street-address': d[self.streetField] + d[self.streetSecondPartField],
                         'facility': d[self.laboratoryNameField],
                         'facility-affiliation': d[self.laboratoryAffiliationField],
                         }
            internship_node = node_dict(the_id=internship_id, **input_dict)
            self.append_id_node(internship_id, internship_node)
            person_email = cleanEmailField(d[self.emailField])
            first_name = d[self.firstNameField]
            last_name = d[self.lastNameField]
            person_id = persons.id_from_something(first_name, last_name, person_email)
            if None == person_id:
                print("ERROR: missing %r (%r %r), creating person node with no interests" % (person_email, first_name, last_name))
                person_id, person_node = addPersonNode(first_name=first_name, last_name=last_name, email=person_email)
                self.append_id_node(person_id, person_node)
            self.append_link(source_id=person_id, target_id=internship_id, label='Did this internship')

def main(args, internships):
    cfg = Config.init_from_file(args.config)
    kernel = RZ_Kernel()
    kernel.db_ctl = dbc.DB_Controller(cfg) # yes, that. FIXME
    globals()['kernel'] = kernel
    globals()['ctx'] = {} # FIXME not logged it - fix later (also, don't do this here, put constructor in kernel)
    StudentCSV(filename=args.student).run()
    for internship in internships:
        StudentInternshipsCSV(filename=internship).run()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--student', required=True)
    parser.add_argument('--config', default=os.path.join(root, 'res', 'etc', 'rhizi-server.conf'))
    args, internships = parser.parse_known_args(sys.argv[1:])
    main(args, internships)