In This Package:

csvrw.py

Go to the documentation of this file.

import os, csv

class CSV(list):
    """
    Reader/writer for .csv files. The contents are stored as a list of dicts.

    :param delimiter: csv field divider
    :param prefix:  string start of comment lines to be ignored, default #Table
    :param descmarker: strings used to identify the field description line 
    :param synth: when defined, add extra field with this name to hold the csv source line number 


    Read usage example::

        src = CSV("$DBWRITERROOT/share/DYB_MC_AD1.txt", delimiter="\\t" )
        src.read()
        for d in src:
            print d

        len(src)
        src[0]
        src[-1]
        src.fieldnames


    On reading an invalid CSV an exception, with error report, is raised::

        src = CSV("$DBWRITERROOT/share/DYB_SAB_AD1.txt", delimiter="\\t" )
        src.read()


    Handling of common csv incorrectnesses is made:

    #. description line fixed up to conform to the delimiter
    #. description line extraneous characters removed (other than fieldnames and delimiters)
    #. removes comments 
 
    Write usage example, field names are obtained from the dict keys::

         out = CSV("/tmp/demo.csv", delimiter="\\t" )
         for d in list_of_dict_datasource:
             out.append(d)
         out.write()

    """
    defaults = dict( delimiter="\t", prefix="#Table", descmarker="#[]", synth="_srcline" )

    def __init__(self, path, **kwargs ):
        """
       """ 
        self._path = path
        self.kwargs = kwargs
        self.stat = dict( hasblank=0 )

    def _fieldnames(self):
        """
        If fieldnames keyword argument is supplied return that otherwise
        return the names of the keys in the first contained dict.  In order to
        control the order of fields, the argument has to be specified.
        """
        fns = self.kwargs.get('fieldnames',None)
        if fns:
            return fns
        if len(self)>0:
            return self[0].keys()     
        return None

    path = property(lambda self:os.path.expanduser(os.path.expandvars(self._path)))
    delimiter = property(lambda self:self.kwargs.get('delimiter',self.defaults.get('delimiter',None)))
    fieldnames = property(_fieldnames, doc=_fieldnames.__doc__)
    descline = property(lambda self:"#" + self.delimiter.join(self.fieldnames) )

    def read(self):
        kwargs = self.kwargs
        delimiter = kwargs.pop('delimiter', self.defaults['delimiter'])
        prefix =    kwargs.pop('prefix', self.defaults['prefix'])
        descmarker = kwargs.pop('descmarker',self.defaults['descmarker'])
        synth =     kwargs.pop('synth',self.defaults['synth'])

        src = Source(open(self.path,"r"), delimiter=delimiter, prefix=prefix, descmarker=descmarker, synth=synth )

        for i,r in enumerate(csv.DictReader(src,delimiter=delimiter)):
            row = Entry(r)
            hasblank = row.hasblank
            if hasblank:
                self.stat['hasblank'] += 1 
            row.update( _hasblank=hasblank )
            self.append(row)
        self.src = src

        if not self.is_valid():
            self.smry()
            raise Exception("CSV.read INVALID .csv :  %r " % self )  


    def is_valid(self):
        return len(self.blanks()) == 0 

    def blanks(self):
        return filter( lambda _:_['_hasblank'] == True, self)

    def smry(self):
        print "source csv...", self.src
        print "CSV ... ", self
        print "irregularities in csv..."
        for blk in self.blanks():
            print blk
        for blk in self.blanks():
            print self.src[int(blk['_srcline'])]
   
    def __repr__(self):
        return "CSV %s %s rows %s stat %r  " % ( self._path, ["NOT VALID","valid"][self.is_valid()], len(self), self.stat )

    def write(self):
        """
        """
        out = open(self.path,"w")
        out.write("#Table CSV.write \n" )  
        out.write( self.descline + "\n")
        writer = csv.DictWriter( out , self.fieldnames , delimiter=self.delimiter )
        for d in self:
            writer.writerow(d)
        out.close()



class Entry(dict):
    hasblank = property(lambda self:len(filter(lambda _:_=="",self.values()))>0)



class Source(list):
    """
    Behaves like a file and holds the original text of the CSV. 
    Applies some fixes to make readable as CSV::

    #. removes comments
    #. normalize the description line to conform to the delimiter
 
    """
    def __init__(self, f, delimiter="\t", prefix="#Table", descmarker="#[]", synth="srcline"):
        """
        :param delimiter: csv field divider
        :param prefix:  string start of lines to be ignored
        :param descmarker: strings used to identify the field description line 
        :param synth: when defined, add extra field with this name to hold the csv source line number 
        """
        self.f = f
        self.prefix = prefix
        self.delimiter = delimiter
        self.descmarker = descmarker
        self.synth = synth

        self.stat = dict(total=0,prefix=0, payload=0, descline=0)
        self.cols = None

    def is_descline(self, line):
        """
        Checks if line contains all of the description markers
        """
        return len(filter(lambda _:_ in line,self.descmarker)) == len(self.descmarker)

    def descline(self, line):
        for c in self.descmarker:
             line = line.replace(c,"")
        return line

    def next(self):
        line = self.f.next()
        self.append(line.rstrip())
        self.stat['total'] += 1
        while line.startswith(self.prefix):
            self.stat['prefix'] += 1
            line = self.f.next()
        else:
            if self.is_descline(line):
                self.stat['descline'] += 1
                if self.synth:
                    line=self.synth + self.delimiter + self.descline(line)
                self.cols = line.rstrip().split(self.delimiter)
            else:
                self.stat['payload'] += 1
                if self.synth: 
                    line = "%d" % (len(self) - 1) + self.delimiter + line 
            return line
    def __iter__(self):
        return self

    def __repr__(self):
        return "Source stat:%r \n cols:%r  " % ( self.stat, self.cols ) 


if __name__ == '__main__':
    paths = "$DBWRITERROOT/share/DYB_SAB_AD1.txt", "$DBWRITERROOT/share/DYB_MC_AD1.txt"

    for path in paths:
        csv_ = CSV(path)
        csv_.read()
        print csv_

---