Top

StarFile module

__copyright = """
PYCIFRW License Agreement (Python License, Version 2)
-----------------------------------------------------

1. This LICENSE AGREEMENT is between the Australian Nuclear Science
and Technology Organisation ("ANSTO"), and the Individual or
Organization ("Licensee") accessing and otherwise using this software
("PyCIFRW") in source or binary form and its associated documentation.

2. Subject to the terms and conditions of this License Agreement,
ANSTO hereby grants Licensee a nonexclusive, royalty-free, world-wide
license to reproduce, analyze, test, perform and/or display publicly,
prepare derivative works, distribute, and otherwise use PyCIFRW alone
or in any derivative version, provided, however, that this License
Agreement and ANSTO's notice of copyright, i.e., "Copyright (c)
2001-2014 ANSTO; All Rights Reserved" are retained in PyCIFRW alone or
in any derivative version prepared by Licensee.

3. In the event Licensee prepares a derivative work that is based on
or incorporates PyCIFRW or any part thereof, and wants to make the
derivative work available to others as provided herein, then Licensee
hereby agrees to include in any such work a brief summary of the
changes made to PyCIFRW.

4. ANSTO is making PyCIFRW available to Licensee on an "AS IS"
basis. ANSTO MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, ANSTO MAKES NO AND
DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYCIFRW WILL NOT
INFRINGE ANY THIRD PARTY RIGHTS.

5. ANSTO SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYCIFRW
FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A
RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYCIFRW, OR ANY
DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.

6. This License Agreement will automatically terminate upon a material
breach of its terms and conditions.

7. Nothing in this License Agreement shall be deemed to create any
relationship of agency, partnership, or joint venture between ANSTO
and Licensee. This License Agreement does not grant permission to use
ANSTO trademarks or trade name in a trademark sense to endorse or
promote products or services of Licensee, or any third party.

8. By copying, installing or otherwise using PyCIFRW, Licensee agrees
to be bound by the terms and conditions of this License Agreement.

"""


from urllib import *         # for arbitrary opening
import re
import copy
# For Python 2.6 or higher compatibility
try: 
    set
except NameError:
    import sets
    set = sets.Set

class StarList(list):
    pass

class StarDict(dict):
    pass


class LoopBlock(object):
    def __init__(self,parent_block,dataname):
        self.loop_no = parent_block.FindLoop(dataname)
        if self.loop_no < 0:
            raise KeyError, '%s is not in a loop structure' % dataname
        self.parent_block = parent_block

    def keys(self):
        return self.parent_block.loops[self.loop_no]

    def values(self):
        return [self.parent_block[a] for a in self.keys()]

    def items(self):
        return zip(self.keys(),self.values())

    def __getitem__(self,dataname):
        if isinstance(dataname,int):   #a packet request
            return self.GetPacket(dataname)
        if dataname in self.keys():
            return self.parent_block[dataname]
        else:
            raise KeyError, '%s not in loop block' % dataname

    def __setitem__(self,dataname,value):
        self.parent_block[dataname] = value
        self.parent_block.AddLoopName(self.keys()[0],dataname)

    def has_key(self,key):
        return key in self.parent_block.loops[self.loop_no]

    def __iter__(self):
        packet_list = zip(*self.values())
        names = self.keys()
        for p in packet_list:
            r = StarPacket(p)
            for n in range(len(names)):
                setattr(r,names[n].lower(),r[n])
            yield r

    # for compatibility
    def __getattr__(self,attname):
        return getattr(self.parent_block,attname)

    def load_iter(self,coords=[]):
        count = 0        #to create packet index 
        while not self.popout:
            # ok, we have a new packet:  append a list to our subloops
            for aloop in self.loops:
                aloop.new_enclosing_packet()
            for iname in self.item_order:
                if isinstance(iname,LoopBlock):       #into a nested loop
                    for subitems in iname.load_iter(coords=coords+[count]):
                        # print 'Yielding %s' % `subitems`
                        yield subitems
                    # print 'End of internal loop'
                else:
                    if self.dimension == 0:
                        # print 'Yielding %s' % `self[iname]`
                        yield self,self[iname]
                    else:
                        backval = self.block[iname]
                        for i in range(len(coords)):
                           # print 'backval, coords: %s, %s' % (`backval`,`coords`)
                           backval = backval[coords[i]]
                        yield self,backval
            count = count + 1      # count packets
        self.popout = False        # reinitialise
        # print 'Finished iterating'
        yield self,'###Blank###'     #this value should never be used

    # an experimental fast iterator for level-1 loops (ie CIF)
    def fast_load_iter(self):
        targets = map(lambda a:self.block[a],self.item_order)
        while targets:
            for target in targets:
                yield self,target

    # Add another list of the required shape to take into account a new outer packet
    def new_enclosing_packet(self):
        if self.dimension > 1:      #otherwise have a top-level list
            for iname in self.keys():  #includes lower levels
                target_list = self[iname]
                for i in range(3,self.dimension): #dim 2 upwards are lists of lists of... 
                    target_list = target_list[-1]
                target_list.append([])
                # print '%s now %s' % (iname,`self[iname]`)

    def recursive_iter(self,dict_so_far={},coord=[]):
        # print "Recursive iter: coord %s, keys %s, dim %d" % (`coord`,`self.block.keys()`,self.dimension)
        my_length = 0
        top_items = self.block.items()
        top_values = self.block.values()       #same order as items
        drill_values = self.block.values()
        for dimup in range(0,self.dimension):  #look higher in the tree
            if len(drill_values)>0:            #this block has values
                drill_values=drill_values[0]   #drill in
            else:
                raise StarError("Malformed loop packet %s" % `top_items[0]`)
        my_length = len(drill_values[0])       #length of 'string' entry
        if self.dimension == 0:                #top level
            for aloop in self.loops:
                for apacket in aloop.recursive_iter():
                    # print "Recursive yielding %s" % `dict(top_items + apacket.items())`
                    prep_yield = StarPacket(top_values+apacket.values())  #straight list
                    for name,value in top_items + apacket.items():
                        setattr(prep_yield,name,value)
                    yield prep_yield
        else:                                  #in some loop
            for i in range(my_length):
                kvpairs = map(lambda a:(a,self.coord_to_group(a,coord)[i]),self.block.keys())
                kvvals = map(lambda a:a[1],kvpairs)   #just values
                # print "Recursive kvpairs at %d: %s" % (i,`kvpairs`)
                if self.loops:
                  for aloop in self.loops:
                    for apacket in aloop.recursive_iter(coord=coord+[i]):
                        # print "Recursive yielding %s" % `dict(kvpairs + apacket.items())`
                        prep_yield = StarPacket(kvvals+apacket.values())
                        for name,value in kvpairs + apacket.items():
                            setattr(prep_yield,name,value)
                        yield prep_yield
                else:           # we're at the bottom of the tree
                    # print "Recursive yielding %s" % `dict(kvpairs)`
                    prep_yield = StarPacket(kvvals)
                    for name,value in kvpairs:
                        setattr(prep_yield,name,value)
                    yield prep_yield

    # small function to use the coordinates. 
    def coord_to_group(self,dataname,coords):
          if not isinstance(dataname,basestring):
             return dataname     # flag inner loop processing
          newm = self[dataname]          # newm must be a list or tuple
          for c in coords:
              # print "Coord_to_group: %s ->" % (`newm`),
              newm = newm[c]
              # print `newm`
          return newm 

    def flat_iterator(self):
        if self.dimension == 0:   
            yield copy.copy(self)
        else:
            my_length = 0
            top_keys = self.block.keys()
            if len(top_keys)>0:
                my_length = len(self.block[top_keys[0]])
            for pack_no in range(my_length):
                yield(self.collapse(pack_no))
            

    def RemoveItem(self,itemname):
        # first check any loops
        loop_no = self.FindLoop(itemname)
        testkey = itemname.lower()
        if self.has_key(testkey):
            del self.block[testkey]
            del self.true_case[testkey]
            # now remove from loop
            if loop_no >= 0:
                self.loops[loop_no].remove(testkey)
                if len(self.loops[loop_no])==0:
                    del self.loops[loop_no]
                    self.item_order.remove(loop_no)
            else:  #will appear in order list
                self.item_order.remove(testkey)
     
    def RemoveLoopItem(self,itemname):
        self.RemoveItem(itemname)

    def GetLoop(self,keyname):
        """Return a LoopBlock object containing keyname"""
        return LoopBlock(self,keyname)

    def GetPacket(self,index):
        thispack = StarPacket([])
        for myitem in self.parent_block.loops[self.loop_no]:
            thispack.append(self[myitem][index])
            setattr(thispack,myitem,thispack[-1])
        return thispack 

    def AddPacket(self,packet):
        for myitem in self.parent_block.loops[self.loop_no]:
            old_values = self.parent_block[myitem]
            old_values.append(packet.__getattribute__(myitem))
            self.parent_block[myitem] = old_values
        
    def GetItemOrder(self):
        return self.parent_block.loops[self.loop_no][:]


    def GetItemOrder(self):
        return self.parent_block.loops[self.loop_no][:]

    def ChangeItemOrder(self,itemname,newpos):
        self.parent_block.loops[self.loop_no].remove(itemname.lower())
        self.parent_block.loops[self.loop_no].insert(newpos,itemname.lower())

    def GetItemPosition(self,itemname):
        import string
        if isinstance(itemname,int):
            # return loop position
            return (-1, self.item_order.index(itemname))
        if not self.has_key(itemname):
            raise ValueError, 'No such dataname %s' % itemname
        testname = itemname.lower()
        if testname in self.item_order:
            return (-1,self.item_order.index(testname))
        loop_no = self.FindLoop(testname)
        loop_pos = self.loops[loop_no].index(testname)
        return loop_no,loop_pos

    def GetLoopNames(self,keyname):
        if keyname in self:
            return self.keys()
        for aloop in self.loops:
            try: 
                return aloop.GetLoopNames(keyname)
            except KeyError:
                pass
        raise KeyError, 'Item does not exist'

    def GetLoopNames(self,keyname):
        """Return all datanames appearing together with [[keyname]]"""
        loop_no = self.FindLoop(keyname)
        if loop_no >= 0:
            return self.loops[loop_no]
        else:
            raise KeyError, '%s is not in any loop' % keyname

    def AddToLoop(self,dataname,loopdata):
        thisloop = self.GetLoop(dataname)
        for itemname,itemvalue in loopdata.items():
            thisloop[itemname] = itemvalue 
        
    def AddToLoop(self,dataname,loopdata):
        # check lengths
        thisloop = self.FindLoop(dataname)
        loop_len = len(self[dataname])
        bad_vals = [a for a in loopdata.items() if len(a[1])!=loop_len]
        if len(bad_vals)>0:
           raise StarLengthError, "Number of values for looped datanames %s not equal to %d" \
               % (`bad_vals`,loop_len)
        self.update(loopdata)
        self.loops[thisloop]+=loopdata.keys()


class StarBlock(object):
    def __init__(self,data = (), maxoutlength=2048, wraplength=80, overwrite=True,
                 characterset='ascii',maxnamelength=-1):
        self.block = {}    #the actual data storage (lower case keys)
        self.loops = {}    #each loop is indexed by a number and contains a list of datanames
        self.item_order = []  #lower case, loops referenced by integer
        self.formatting_hints = {}
        self.true_case = {} #transform lower case to supplied case
        self.provide_value = False   #prefer string version always
        self.dictionary = None      #DDLm dictionary
        self.popout = False         #used during load iteration
        self.curitem = -1           #used during iteration
        self.maxoutlength = maxoutlength
        self.setmaxnamelength(maxnamelength)  #to enforce CIF limit of 75 characters
        self.wraplength = wraplength
        self.overwrite = overwrite
        self.characterset = characterset
        if self.characterset == 'ascii':
            self.char_check = re.compile("[][ \n\r\t!%&\(\)*+,./:<=>?@0-9A-Za-z\\\\^`{}\|~\"#$';_-]+",re.M)
        elif self.characterset == 'unicode':
            self.char_check = re.compile(u"[][ \n\r\t!%&\(\)*+,./:<=>?@0-9A-Za-z\\\\^`{}\|~\"#$';_\u00A0-\uD7FF\uE000-\uFDCF\uFDF0-\uFFFD\U00010000-\U0010FFFD-]+",re.M)
        else:
            raise StarError("No character set specified")
        if isinstance(data,(tuple,list)):
            for item in data:
                self.AddLoopItem(item)
        elif isinstance(data,StarBlock):
            self.block = data.block.copy() 
            self.item_order = data.item_order[:]
            self.true_case = data.true_case.copy()
            # loops as well
            self.loops = data.loops.copy()

    def setmaxnamelength(self,maxlength):
        """Set the maximum allowable dataname length (-1 for no check)"""
        self.maxnamelength = maxlength
        if maxlength > 0:
            bad_names = [a for a in self.keys() if len(a)>self.maxnamelength]
            if len(bad_names)>0:
                raise StarError, 'Datanames too long: ' + `bad_names`

    def __str__(self):
        return self.printsection()

    def __setitem__(self,key,value):
        if key == "saves":
            raise StarError("""Setting the saves key is deprecated. Add the save block to
    an enclosing block collection (e.g. CIF or STAR file) with this block as child""")
        self.AddItem(key,value)

    def __getitem__(self,key):
        if key == "saves":
            raise StarError("""The saves key is deprecated. Access the save block from
    the enclosing block collection (e.g. CIF or STAR file object)""") 
        try:
           rawitem,is_value = self.GetFullItemValue(key)
        except KeyError:
           if self.dictionary:
               # send the dictionary the required key and a pointer to us
               new_value = self.dictionary.derive_item(key,self,store_value=True)
               print 'Set %s to derived value %s' % (key, `new_value`)
               return new_value
           else:
               raise KeyError, 'No such item: %s' % key
        # we now have an item, we can try to convert it to a number if that is appropriate
        # note numpy values are never stored but are converted to lists
        if not self.dictionary or not self.dictionary.has_key(key): return rawitem
        print '%s: is_value %s provide_value %s value %s' % (key,`is_value`,`self.provide_value`,`rawitem`)
        if is_value:
            if self.provide_value: return rawitem
            else:
               print 'Turning %s into string' % `rawitem`
               return self.convert_to_string(key)
        else:    # a string
            if self.provide_value and rawitem != '?' and rawitem != ".":
                return self.dictionary.change_type(key,rawitem)
            return rawitem   #can't do anything
           
    def __delitem__(self,key):
        self.RemoveItem(key)

    def __len__(self):
        blen = len(self.block)
        return blen    

    def __nonzero__(self):
        if self.__len__() > 0: return 1
        return 0

    # keys returns all internal keys
    def keys(self):
        return self.block.keys()    #always lower case

    def values(self):
        return [self[a] for a in self.keys()]

    def items(self):
        return [a for a in zip(self.keys(),self.values())]

    def has_key(self,key):
        if isinstance(key,basestring) and key.lower() in self.keys():
            return 1
        return 0

    def get(self,key,default=None):
        if self.has_key(key):
            retval = self.__getitem__(key)
        else:
            retval = default
        return retval

    def clear(self):
        self.block = {}
        self.loops = {}
        self.item_order = []
        self.true_case = {}

    # doesn't appear to work
    def copy(self):
        newcopy = StarBlock()
        newcopy.block = self.block.copy()
        newcopy.loops = []
        newcopy.item_order = self.item_order[:]
        newcopy.true_case = self.true_case.copy()
        newcopy.loops = self.loops.copy()
    #    return self.copy.im_class(newcopy)   #catch inheritance
        return newcopy
     
    def update(self,adict):
        for key in adict.keys():
            self.AddItem(key,adict[key])

    def GetItemPosition(self,itemname):
        import string
        if isinstance(itemname,int):
            # return loop position
            return (-1, self.item_order.index(itemname))
        if not self.has_key(itemname):
            raise ValueError, 'No such dataname %s' % itemname
        testname = itemname.lower()
        if testname in self.item_order:
            return (-1,self.item_order.index(testname))
        loop_no = self.FindLoop(testname)
        loop_pos = self.loops[loop_no].index(testname)
        return loop_no,loop_pos

    def ChangeItemOrder(self,itemname,newpos):
        """Move itemname to newpos"""
        if isinstance(itemname,basestring):
            true_name = itemname.lower()
        else:
            true_name = itemname
        self.item_order.remove(true_name)
        self.item_order.insert(newpos,true_name)

    def GetItemOrder(self):
        return self.item_order[:]

    def AddItem(self,key,value,precheck=False):
        if not isinstance(key,basestring):
             raise TypeError, 'Star datanames are strings only (got %s)' % `key`
        if not precheck:
             self.check_data_name(key,self.maxnamelength)    # make sure no nasty characters   
        # check for overwriting
        if self.has_key(key):
             if not self.overwrite:
                 raise StarError( 'Attempt to insert duplicate item name %s' % key)
        # put the data in the right container
        regval,empty_val = self.regularise_data(value)
        # check for pure string data
        pure_string = check_stringiness(regval)
        if not precheck:
            self.check_item_value(regval)
        # update ancillary information first
        lower_key = key.lower()
        if not self.has_key(lower_key) and self.FindLoop(lower_key)<0:      #need to add to order
            self.item_order.append(lower_key)
        # always remove from our case table in case the case is different
        else:
            del self.true_case[lower_key]
        self.true_case[lower_key] = key
        if pure_string:
            self.block.update({lower_key:[regval,empty_val]})
        else:
            self.block.update({lower_key:[empty_val,regval]})  
            
    def AddLoopItem(self,incomingdata,precheck=False,maxlength=-1):
        # print "Received data %s" % `incomingdata`
        # we accept tuples, strings, lists and dicts!!
        # Direct insertion: we have a string-valued key, with an array
        # of values -> single-item into our loop
        if isinstance(incomingdata[0],(tuple,list)):
           # a whole loop
           keyvallist = zip(incomingdata[0],incomingdata[1])
           for key,value in keyvallist:
               self.AddItem(key,value)
           self.CreateLoop(incomingdata[0])
        elif not isinstance(incomingdata[0],basestring):
             raise TypeError, 'Star datanames are strings only (got %s)' % `incomingdata[0]`
        else:
            self.AddItem(incomingdata[0],incomingdata[1])

    def check_data_name(self,dataname,maxlength=-1): 
        if maxlength > 0:
            self.check_name_length(dataname,maxlength)
        if dataname[0]!='_':
            raise StarError( 'Dataname ' + dataname + ' does not begin with _')
        if self.characterset=='ascii':
            if len (filter (lambda a: ord(a) < 33 or ord(a) > 126, dataname)) > 0:
                raise StarError( 'Dataname ' + dataname + ' contains forbidden characters')
        else:
            # print 'Checking %s for unicode characterset conformance' % dataname
            if len (filter (lambda a: ord(a) < 33, dataname)) > 0:
                raise StarError( 'Dataname ' + dataname + ' contains forbidden characters (below code point 33)')
            if len (filter (lambda a: ord(a) > 126 and ord(a) < 160, dataname)) > 0:
                raise StarError( 'Dataname ' + dataname + ' contains forbidden characters (between code point 127-159)')
            if len (filter (lambda a: ord(a) > 0xD7FF and ord(a) < 0xE000, dataname)) > 0:
                raise StarError( 'Dataname ' + dataname + ' contains unsupported characters (between U+D800 and U+E000)')
            if len (filter (lambda a: ord(a) > 0xFDCF and ord(a) < 0xFDF0, dataname)) > 0:
                raise StarError( 'Dataname ' + dataname + ' contains unsupported characters (between U+FDD0 and U+FDEF)')
            if len (filter (lambda a: ord(a) == 0xFFFE or ord(a) == 0xFFFF, dataname)) > 0:
                raise StarError( 'Dataname ' + dataname + ' contains unsupported characters (U+FFFE and/or U+FFFF)')
            if len (filter (lambda a: ord(a) > 0x10000 and (ord(a) & 0xE == 0xE) , dataname)) > 0:
                print '%s fails' % dataname
                for a in dataname: print '%x' % ord(a),
                print
                raise StarError( u'Dataname ' + dataname + u' contains unsupported characters (U+xFFFE and/or U+xFFFF)')

    def check_name_length(self,dataname,maxlength):
        if len(dataname)>maxlength:
            raise StarError( 'Dataname %s exceeds maximum length %d' % (dataname,maxlength))
        return

    def check_item_value(self,item):
        test_item = item
        if not isinstance(item,(list,dict,tuple)):
           test_item = [item]         #single item list
        def check_one (it):
            if isinstance(it,basestring):
                if it=='': return
                me = self.char_check.match(it)            
                if not me:
                    print "Fail value check: %s" % it
                    raise StarError, u'Bad character in %s' % it
                else:
                    if me.span() != (0,len(it)):
                        print "Fail value check, match only %d-%d in string %s" % (me.span()[0],me.span()[1],`it`)
                        raise StarError,u'Data item "' + `it` +  u'"... contains forbidden characters'
        map(check_one,test_item)

    def regularise_data(self,dataitem):
        """Place dataitem into a list if necessary"""
        from numbers import Number
        if isinstance(dataitem,(Number,basestring,StarList,StarDict)):
            return dataitem,None
        if isinstance(dataitem,(tuple,list)):
            return dataitem,[None]*len(dataitem)
        # so try to make into a list
        try:
            regval = list(dataitem)
        except TypeError, value:
            raise StarError( str(dataitem) + ' is wrong type for data value\n' )
        return regval,[None]*len(regval)
        
    def RemoveItem(self,itemname):
        # first check any loops
        loop_no = self.FindLoop(itemname)
        testkey = itemname.lower()
        if self.has_key(testkey):
            del self.block[testkey]
            del self.true_case[testkey]
            # now remove from loop
            if loop_no >= 0:
                self.loops[loop_no].remove(testkey)
                if len(self.loops[loop_no])==0:
                    del self.loops[loop_no]
                    self.item_order.remove(loop_no)
            else:  #will appear in order list
                self.item_order.remove(testkey)
     
    def RemoveLoopItem(self,itemname):
        self.RemoveItem(itemname)

    def GetItemValue(self,itemname):
        """Return value of itemname"""
        return self.GetFullItemValue(itemname)[0]

    def GetFullItemValue(self,itemname):
        """Return value of itemname and whether or not it is a native value"""
        try:
            s,v = self.block[itemname.lower()]
        except KeyError:
            raise KeyError, 'Itemname %s not in datablock' % itemname
        # prefer string value unless all are None
        # are we a looped value?
        if not isinstance(s,(tuple,list)) or isinstance(s,StarList):
            if s is not None or (isinstance(s,StarList) and None not in s): 
                return s,False    #a string value
            else:
                return v,not isinstance(v,StarList)  #a StarList is not calculation-ready
        elif None not in s: 
            return s,False     #a list of string values
        else: 
            if len(v)>0:   
                return v,not isinstance(v[0],StarList)
            return v,True

    def CreateLoop(self,datanames,order=-1):
           """[[datanames]] is a list of datanames that together form a loop.  They should
           all contain the same number of elements (possibly 0). If [[order]] is given, the loop will
           appear at this position in the block. A loop counts as a single position."""
           # check lengths: these datanames should exist
           listed_values = [a for a in datanames if isinstance(self[a],list) and not isinstance(self[a],StarList)]
           if len(listed_values) == len(datanames):
               len_set = set([len(self[a]) for a in datanames])
               if len(len_set)>1:
                   raise ValueError, 'Request to loop datanames %s with different lengths: %s' % (`datanames`,`len_set`)
           elif len(listed_values) != 0:
               raise ValueError, 'Request to loop datanames where some are single values and some are not'
           if len(self.loops)>0:
               loopno = max(self.loops.keys()) + 1
           else:
               loopno = 1
           # store as lower case
           lc_datanames = [d.lower() for d in datanames]
           # remove these datanames from all other loops
           [self.loops[a].remove(b) for a in self.loops for b in lc_datanames if b in self.loops[a]]
           self.loops[loopno] = list(lc_datanames)
           if order >= 0:
               self.item_order.insert(order,loopno)
           else:
               self.item_order.append(loopno)
           # remove these datanames from item ordering
           self.item_order = [a for a in self.item_order if a not in lc_datanames] 
          
    def AddLoopName(self,oldname, newname):
        """Add [[newname]] to the loop containing [[oldname]]"""
        loop_no = self.FindLoop(oldname)
        if loop_no < 0:
            raise KeyError, '%s not in loop' % oldname
        if newname in self.loops[loop_no]:
            return
        # check length
        loop_len = len(self[oldname])
        if len(self[newname]) != loop_len:
            raise ValueError, 'Mismatch of loop column lengths for %s: should be %d' % (newname,loop_len)
        # remove from any other loops
        [self.loops[a].remove(newname) for a in self.loops if newname in self.loops[a]]
        # and add to this loop
        self.loops[loop_no].append(newname)
        
    def FindLoop(self,keyname):
        """Find the loop that contains keyname and return its numerical index,-1 if not present"""
        loop_no = [a for a in self.loops.keys() if keyname.lower() in self.loops[a]]
        if len(loop_no)>0:
            return loop_no[0]
        else:
            return -1

    def GetLoop(self,keyname):
        """Return a LoopBlock object containing keyname"""
        return LoopBlock(self,keyname)

    def GetLoopNames(self,keyname):
        if keyname in self:
            return self.keys()
        for aloop in self.loops:
            try: 
                return aloop.GetLoopNames(keyname)
            except KeyError:
                pass
        raise KeyError, 'Item does not exist'

    def GetLoopNames(self,keyname):
        """Return all datanames appearing together with [[keyname]]"""
        loop_no = self.FindLoop(keyname)
        if loop_no >= 0:
            return self.loops[loop_no]
        else:
            raise KeyError, '%s is not in any loop' % keyname

    def AddLoopName(self,oldname, newname):
        """Add [[newname]] to the loop containing [[oldname]]"""
        loop_no = self.FindLoop(oldname)
        if loop_no < 0:
            raise KeyError, '%s not in loop' % oldname
        if newname in self.loops[loop_no]:
            return
        # check length
        loop_len = len(self[oldname])
        if len(self[newname]) != loop_len:
            raise ValueError, 'Mismatch of loop column lengths for %s: should be %d' % (newname,loop_len)
        # remove from any other loops
        [self.loops[a].remove(newname) for a in self.loops if newname in self.loops[a]]
        # and add to this loop
        self.loops[loop_no].append(newname)
        
    def AddToLoop(self,dataname,loopdata):
        thisloop = self.GetLoop(dataname)
        for itemname,itemvalue in loopdata.items():
            thisloop[itemname] = itemvalue 
        
    def AddToLoop(self,dataname,loopdata):
        # check lengths
        thisloop = self.FindLoop(dataname)
        loop_len = len(self[dataname])
        bad_vals = [a for a in loopdata.items() if len(a[1])!=loop_len]
        if len(bad_vals)>0:
           raise StarLengthError, "Number of values for looped datanames %s not equal to %d" \
               % (`bad_vals`,loop_len)
        self.update(loopdata)
        self.loops[thisloop]+=loopdata.keys()

    def Loopify(self,datanamelist):
        self.CreateLoop(datanamelist)

    def RemoveKeyedPacket(self,keyname,keyvalue):
        packet_coord = list(self[keyname]).index(keyvalue)
        loopnames = self.GetLoopNames(keyname)
        for dataname in loopnames:
            self.block[dataname][0] = list(self.block[dataname][0])
            del self.block[dataname][0][packet_coord]
            self.block[dataname][1] = list(self.block[dataname][1])
            del self.block[dataname][1][packet_coord]

    def GetKeyedPacket(self,keyname,keyvalue,no_case=False):
        """Return the loop packet where [[keyname]] has value [[keyvalue]]. Ignore case if no_case is true"""
        #print "Looking for %s in %s" % (keyvalue, self.parent_block[keyname])
        my_loop = self.GetLoop(keyname)
        if no_case:
           one_pack= [a for a in my_loop if getattr(a,keyname).lower()==keyvalue.lower()]
        else:
           one_pack= [a for a in my_loop if getattr(a,keyname)==keyvalue]
        if len(one_pack)!=1:
            raise ValueError, "Bad packet key %s = %s: returned %d packets" % (keyname,keyvalue,len(one_pack))
        #print "Keyed packet: %s" % one_pack[0]
        return one_pack[0]

    def GetKeyedSemanticPacket(self,keyvalue,cat_id):
        """Return a complete packet for category cat_id"""
        target_keys = self.dictionary.cat_key_table[cat_id]
        p = StarPacket()
        # set case-sensitivity flag
        lcase = False
        if self.dictionary[target_keys[0]]['_type.contents'] in ['Code','Tag','Name']:
            lcase = True
        for cat_key in target_keys:
            try:
                extra_packet = self.GetKeyedPacket(cat_key,keyvalue,no_case=lcase)
            except KeyError:        #try to create the key
                key_vals = self[cat_key]   #will create a key column
            p.merge_packet(self.GetKeyedPacket(cat_key,keyvalue,no_case=lcase))
        # the following attributes used to calculate missing values
        p.key = target_keys[0]
        p.cif_dictionary = self.dictionary
        p.fulldata = self
        return p


    def SetOutputLength(self,wraplength=80,maxoutlength=2048):
        if wraplength > maxoutlength:
            raise StarError("Wrap length (requested %d) must be <= Maximum line length (requested %d)" % (wraplength,maxoutlength))
        self.wraplength = wraplength
        self.maxoutlength = maxoutlength

    def printsection(self,instring='',ordering=[],blockstart="",blockend="",indent=0):
        import string
        # first make an ordering
        self.create_ordering(ordering)
        # now do it...
        if not instring:
            outstring = CIFStringIO(target_width=80)       # the returned string
        else:
            outstring = instring
        # print loop delimiter
        outstring.write(blockstart,canbreak=True)
        while len(self.output_order)>0:
           #print "Remaining to output " + `self.output_order`
           itemname = self.output_order.pop(0)
           item_spec = [i for i in ordering if i['dataname'].lower()==itemname.lower()]
           if len(item_spec)>0:
               col_pos = item_spec[0].get('column',-1)
           else:
               col_pos = -1
               item_spec = {}
               if not isinstance(itemname,int):  #no loop
                   if col_pos < 0: col_pos = 40
                   outstring.set_tab(col_pos)
                   itemvalue = self[itemname]
                   outstring.write(self.true_case[itemname],mustbreak=True,do_tab=False)
                   outstring.write(' ',canbreak=True,do_tab=False)    #space after itemname
                   self.format_value(itemvalue,outstring,hints=item_spec)
               else:   # we are asked to print a loop block
                    #first make sure we have sensible coords.  Length should be one
                    #less than the current dimension
                    outstring.set_tab(10)       #guess this is OK?
                    outstring.write(' '*indent,mustbreak=True,do_tab=False); outstring.write('loop_\n',do_tab=False)
                    self.format_names(outstring,indent+2,loop_no=itemname)
                    self.format_packets(outstring,indent+2,loop_no=itemname)
        else:
            returnstring = outstring.getvalue()
        outstring.close()
        return returnstring

    def format_names(self,outstring,indent=0,loop_no=-1):
        """Print datanames from [[loop_no]] one per line"""
        temp_order = self.loops[loop_no][:]   #copy
        while len(temp_order)>0:
            itemname = temp_order.pop(0)
            outstring.write(' ' * indent,do_tab=False) 
            outstring.write(self.true_case[itemname],do_tab=False)
            outstring.write("\n",do_tab=False)

    def format_packets(self,outstring,indent=0,loop_no=-1):
       import cStringIO
       import string
       alldata = [self[a] for a in self.loops[loop_no]]
       print 'Alldata: %s' % `alldata`
       packet_data = apply(zip,alldata)
       print 'Packet data: %s' % `packet_data`
       for position in range(len(packet_data)):
           for point in range(len(packet_data[position])):
               datapoint = packet_data[position][point]
               packstring = self.format_packet_item(datapoint,indent,outstring)
           outstring.write("\n",do_tab=False)
               
    def format_packet_item(self,pack_item,indent,outstring):
           # print 'Formatting %s' % `pack_item`
           if isinstance(pack_item,basestring):
               outstring.write(self._formatstring(pack_item)) 
           else: 
               self.format_value(pack_item,outstring)
           outstring.write(' ',canbreak=True,do_tab=False)

    def _formatstring(self,instring,delimiter=None,standard='CIF1',indent=0,
        lbprotocol=True,pref_protocol=True):
        import string
        if standard == 'CIF2':
            allowed_delimiters = set(['"',"'",";",None,'"""',"'''"])
        else:
            allowed_delimiters = set(['"',"'",";",None])
        if len(instring)==0: allowed_delimiters.difference_update([None])
        if len(instring) > (self.maxoutlength-2) or '\n' in instring:
                allowed_delimiters.intersection_update([";","'''",'"""'])
        if ' ' in instring or '\t' in instring or '\v' in instring or '_' in instring or ',' in instring:
                allowed_delimiters.difference_update([None])
        if '"' in instring: allowed_delimiters.difference_update(['"',None])
        if "'" in instring: allowed_delimiters.difference_update(["'",None])
        out_delimiter = ";"  #default (most conservative)
        if delimiter in allowed_delimiters:
            out_delimiter = delimiter
        elif "'" in allowed_delimiters: out_delimiter = "'"
        elif '"' in allowed_delimiters: out_delimiter = '"'
        if out_delimiter in ['"',"'",'"""',"'''"]: return out_delimiter + instring + out_delimiter
        elif out_delimiter is None: return instring
        # we are left with semicolon strings
        outstring = "\n;"
        # if there are returns in the string, try to work with them
        while 1:
            retin = string.find(instring,'\n')+1
            if retin < self.maxoutlength and retin > 0:      # honour this break
                outstring = outstring + instring[:retin]
                instring = instring[retin:]
            elif len(instring)0:
               self.format_value(itemvalue[0],stringsink)
               for listval in itemvalue[1:]:
                  print 'Formatting %s' % `listval`
                  stringsink.write(', ',do_tab=False)
                  self.format_value(listval,stringsink,compound=True)
           stringsink.write(']',unindent=True)
        elif isinstance(itemvalue,StarDict):
           stringsink.set_tab(0)
           stringsink.write('{',newindent=True,mustbreak=compound)  #start a new line inside
           items = itemvalue.items()
           if len(items)>0:
               stringsink.write("'"+items[0][0]+"'"+':',canbreak=True)
               self.format_value(items[0][1],stringsink)
               for key,value in items[1:]:
                   stringsink.write(', ')
                   stringsink.write("'"+key+"'"+":",canbreak=True)
                   self.format_value(value,stringsink)   #never break between key and value
           stringsink.write('}',unindent=True)
        else: 
           stringsink.write(str(itemvalue),canbreak=True)   #numbers

    def process_template(self,template_string):
        """Process a template datafile to formatting instructions"""
        template_as_cif = StarFile(StringIO(template_string),grammar="DDLm").first_block()
        #template_as_lines = template_string.split("\n")
        #template_as_lines = [l for l in template_as_lines if len(l)>0 and l[0]!='#']
        #template_as_lines = [l for l in template_as_lines if l.split()[0] != 'loop_']
        #template_full_lines = dict([(l.split()[0],l) for l in template_as_lines if len(l.split())>0])
        self.form_hints = []   #ordered array of hint dictionaries
        for item in template_as_cif.item_order:  #order of input
            if not isinstance(item,int):    #not nested
                hint_dict = {"dataname":item}
                # find the line in the file
                start_pos = re.search("(^[ \t]*" + item + "[ \t\n]+)(?P([\S]+)|(^;))",template_string,re.I|re.M)
                if start_pos.group("spec") != None:
                    spec_pos = start_pos.start("spec")-start_pos.start(0)
                    spec_char = template_string[start_pos.start("spec")]
                    if spec_char in '\'";':
                        hint_dict.update({"delimiter":spec_char})
                    if spec_char != ";":   #so we need to work out the column number
                        hint_dict.update({"column":spec_pos})
                print '%s: %s' % (item,`hint_dict`)
                self.form_hints.append(hint_dict)
            else:           #loop block
                testnames = template_as_cif.loops[item]
                total_items = len(template_as_cif.loops[item])
                testname = testnames[0]
                #find the loop spec line in the file
                loop_regex = "(^[ \t]*loop_[ \t\n\r]+" + testname + "([ \t\n\r]+_[\S]+){%d}[ \t]*$(?P(.(?!_loop|_[\S]+))*))" % (total_items - 1)
                loop_line = re.search(loop_regex,template_string,re.I|re.M|re.S)
                loop_so_far = loop_line.end()
                packet_text = loop_line.group('packet')
                packet_regex = "[ \t]*(?P(?P'([^\n\r\f']*)'+)|(?P\"([^\n\r\"]*)\"+)|(?P[^\s]+))"
                packet_pos = re.finditer(packet_regex,packet_text)
                line_end_pos = re.finditer("^",packet_text,re.M)
                next_end = line_end_pos.next().end()
                last_end = next_end
                for loopname in testnames:
                    hint_dict = {"dataname":loopname}
                    thismatch = packet_pos.next()
                    while thismatch.start('all') > next_end:
                        try: 
                            last_end = next_end
                            next_end = line_end_pos.next().start()
                            print 'next end %d' % next_end
                        except StopIteration:
                            pass 
                    print 'Start %d, last_end %d' % (thismatch.start('all'),last_end)
                    col_pos = thismatch.start('all') - last_end
                    if thismatch.group('none') is None:
                        hint_dict.update({'delimiter':thismatch.groups()[0][0]})
                    hint_dict.update({'column':col_pos})
                    print '%s: %s' % (loopname,`hint_dict`)
                    self.form_hints.append(hint_dict)
        return

    def create_ordering(self,order_dict):
        """Create a canonical ordering that includes loops using our formatting hints dictionary"""
        requested_order = [i['dataname'] for i in order_dict]
        new_order = []
        for item in requested_order:
           if isinstance(item,basestring) and item.lower() in self.item_order:
               new_order.append(item.lower())
           elif self.has_key(item):    #in a loop somewhere
               target_loop = self.FindLoop(item)
               if target_loop not in new_order:
                   new_order.append(target_loop)
        extras = [i for i in self.item_order if i not in new_order]
        self.output_order = new_order + extras
        print 'Final order: ' + `self.output_order`

    def convert_to_string(self,dataname):
        """Convert values held in dataname value fork to string version"""
        v,is_value = self.GetFullItemValue(dataname)
        if not is_value:
            return v
        if check_stringiness(v): return v   #already strings
        # TODO...something else
        return v


    def merge(self,new_block,mode="strict",match_att=[],match_function=None,
                   rel_keys = []):
        if mode == 'strict':
           for key in new_block.keys(): 
               if self.has_key(key) and key not in match_att:
                  raise CifError( "Identical keys %s in strict merge mode" % key)
               elif key not in match_att:           #a new dataname
                   self[key] = new_block[key]
           # we get here if there are no keys in common, so we can now copy
           # the loops and not worry about overlaps
           for one_loop in new_block.loops.values():
               self.CreateLoop(one_loop)
           # we have lost case information
           self.true_case.update(new_block.true_case)
        elif mode == 'replace':
           newkeys = new_block.keys()
           for ma in match_att:
              try:
                   newkeys.remove(ma)        #don't touch the special ones
              except ValueError:
                   pass
           for key in new_block.keys(): 
                  if isinstance(key,basestring):
                      self[key] = new_block[key] 
           # creating the loop will remove items from other loops
           for one_loop in new_block.loops.values():
               self.CreateLoop(one_loop)
           # we have lost case information
           self.true_case.update(new_block.true_case)
        elif mode == 'overlay':
           print 'Overlay mode, current overwrite is %s' % self.overwrite
           raise StarError, 'Overlay block merge mode not implemented'
           save_overwrite = self.overwrite
           self.overwrite = True
           for attribute in new_block.keys():
               if attribute in match_att: continue      #ignore this one
               new_value = new_block[attribute]
               #non-looped items
               if new_block.FindLoop(attribute)<0:     #not looped
                  self[attribute] = new_value 
           my_loops = self.loops.values()
           perfect_overlaps = [a for a in new_block.loops if a in my_loops]
           for po in perfect_overlaps:
              loop_keys = [a for a in po if a in rel_keys]  #do we have a key?
              try:
                  newkeypos = map(lambda a:newkeys.index(a),loop_keys)
                  newkeypos = newkeypos[0]      #one key per loop for now
                  loop_keys = loop_keys[0] 
              except (ValueError,IndexError):
                  newkeypos = []
                  overlap_data = map(lambda a:listify(self[a]),overlaps) #old packet data
                  new_data = map(lambda a:new_block[a],overlaps) #new packet data
                  packet_data = transpose(overlap_data)
                  new_p_data = transpose(new_data)
                  # remove any packets for which the keys match between old and new; we
                  # make the arbitrary choice that the old data stays
                  if newkeypos:
                      # get matching values in new list
                      print "Old, new data:\n%s\n%s" % (`overlap_data[newkeypos]`,`new_data[newkeypos]`)
                      key_matches = filter(lambda a:a in overlap_data[newkeypos],new_data[newkeypos])
                      # filter out any new data with these key values
                      new_p_data = filter(lambda a:a[newkeypos] not in key_matches,new_p_data)
                      if new_p_data:
                          new_data = transpose(new_p_data)
                      else: new_data = []
                  # wipe out the old data and enter the new stuff
                  byebyeloop = self.GetLoop(overlaps[0])
                  # print "Removing '%s' with overlaps '%s'" % (`byebyeloop`,`overlaps`)
                  # Note that if, in the original dictionary, overlaps are not
                  # looped, GetLoop will return the block itself.  So we check
                  # for this case...
                  if byebyeloop != self:
                      self.remove_loop(byebyeloop)
                  self.AddLoopItem((overlaps,overlap_data))  #adding old packets
                  for pd in new_p_data:                             #adding new packets
                     if pd not in packet_data:
                        for i in range(len(overlaps)):
                            #don't do this at home; we are appending
                            #to something in place
                            self[overlaps[i]].append(pd[i]) 
           self.overwrite = save_overwrite

    def assign_dictionary(self,dic):
        if not dic.diclang=="DDLm":
            print "Warning: ignoring dictionary %s" % dic.dic_as_cif.my_uri
            return
        self.dictionary = dic

    def unassign_dictionary(self):
        """Remove dictionary-dependent behaviour"""
        self.dictionary = None



class StarPacket(list):
    def merge_packet(self,incoming):
        """Merge contents of incoming packet with this packet"""
        new_attrs = [a for a in dir(incoming) if a[0] == '_' and a[1] != "_"]
        self.append(incoming)
        for na in new_attrs:
            setattr(self,na,getattr(incoming,na))

    def __getattr__(self,att_name):
        """Derive a missing attribute"""
        if att_name.lower() in self.__dict__:
            return getattr(self,att_name.lower())
        if att_name in ('cif_dictionary','fulldata','key'):
            raise AttributeError, 'Programming error: cannot compute value of %s' % att_name
        d = self.cif_dictionary
        c = self.fulldata
        k = self.key
        d.derive_item(att_name,c,store_value=True)
        # 
        # now pick out the new value
        keyval = getattr(self,k)
        full_pack = c.GetKeyedPacket(k,keyval)
        return getattr(full_pack,att_name)
        
class BlockCollection(object):
    def __init__(self,datasource=None,standard='CIF',blocktype = StarBlock,
                 characterset='ascii',scoping='instance',parent_id=None,**kwargs):
        import collections
        self.dictionary = {}
        self.standard = standard
        self.lower_keys = set()           # short_cuts
        self.renamed = {}
        self.characterset = characterset
        self.PC = collections.namedtuple('PC',['block_id','parent'])
        self.child_table = {}
        self.visible_keys = []            # for efficiency
        self.parent_id = parent_id
        self.scoping = scoping  #will trigger setting of child table
        self.blocktype = blocktype
        if isinstance(datasource,BlockCollection):
            self.merge_fast(datasource)
            self.scoping = scoping   #reset visibility
        elif isinstance(datasource,dict):
            for key,value in datasource.items():
                 self[key]= value
        self.header_comment = ''
     
    def unlock(self):
        """Allow overwriting of all blocks in this collection"""
        for a in self.lower_keys:
            self[a].overwrite=True

    def lock(self):
        """Disallow overwriting for all blocks in this collection"""
        for a in self.lower_keys:
            self[a].overwrite = False

    def __str__(self):
        return self.WriteOut()

    def __setitem__(self,key,value):
        self.NewBlock(key,value,parent=None)

    def __getitem__(self,key):
        if isinstance(key,basestring):
           lowerkey = key.lower()
           if lowerkey in self.lower_keys:
               return self.dictionary[lowerkey]
           #print 'Visible keys:' + `self.visible_keys`
           #print 'All keys' + `self.lower_keys`
           #print 'Child table' + `self.child_table`
           raise KeyError,'No such item %s' % key

    # we have to get an ordered list of the current keys,
    # as we'll have to delete one of them anyway.
    # Deletion will delete any key regardless of visibility

    def __delitem__(self,key):
        dummy = self[key]   #raise error if not present
        lowerkey = key.lower()
        # get rid of all children recursively as well
        children = [a[0] for a in self.child_table.items() if a[1].parent == lowerkey]
        for child in children:
            del self[child]   #recursive call
        del self.dictionary[lowerkey]
        del self.child_table[lowerkey]
        try:
            self.visible_keys.remove(lowerkey)
        except KeyError:
            pass
        self.lower_keys.remove(lowerkey)

    def __len__(self):
        return len(self.visible_keys)

    def __contains__(self,item):
        """Support the 'in' operator"""
        return self.has_key(item)

    # We iterate over all visible
    def __iter__(self):
        for one_block in self.keys():
            yield self[one_block]

    # TODO: handle different case
    def keys(self):
        return self.visible_keys

    # changes to take case independence into account
    def has_key(self,key):
        if not isinstance(key,basestring): return 0
        if key.lower() in self.visible_keys:
            return 1
        return 0

    def get(self,key,default=None):
        if self.has_key(key):     # take account of case
            return self.__getitem__(key)
        else:
            return default

    def clear(self):
        self.dictionary.clear()
        self.lower_keys = set()
        self.child_table = {}
        self.visible_keys = []

    def copy(self):   
        newcopy = self.dictionary.copy()  #all blocks
        newcopy = BlockCollection('',newcopy,parent_id=self.parent_id)
        newcopy.child_table = self.child_table.copy()
        newcopy.lower_keys = self.lower_keys
        newcopy.characterset = self.characterset
        newcopy.scoping = self.scoping  #this sets visible keys
        return newcopy

    def update(self,adict):
        for key in adict.keys():
            self[key] = adict[key]

    def items(self):
        return [(a,self[a]) for a in self.keys()]

    def first_block(self):
        """Return the 'first' block.  This is not necessarily the first block in the file."""
        if self.keys():
            return self[self.keys()[0]]

    def NewBlock(self,blockname,blockcontents=None,fix=True,parent=None):
        if blockcontents is None:
            blockcontents = StarBlock()
        if self.standard == "CIF":
            blockcontents.setmaxnamelength(75)
        if len(blockname)>75:
                 raise StarError , 'Blockname %s is longer than 75 characters' % blockname
        if fix:
            newblockname = re.sub('[  \t]','_',blockname)
        else: newblockname = blockname
        new_lowerbn = newblockname.lower()
        if new_lowerbn in self.lower_keys:
            if self.standard is not None:    #already there
               toplevelnames = [a[0] for a in self.child_table.items() if a[1].parent==None]
               if parent is None and new_lowerbn not in toplevelnames:  #can give a new key to this one
                  while new_lowerbn in self.lower_keys: new_lowerbn = new_lowerbn + '+'
               elif parent is not None and new_lowerbn in toplevelnames: #can fix a different one
                  replace_name = new_lowerbn            
                  while replace_name in self.lower_keys: replace_name = replace_name + '+'
                  self._rekey(new_lowerbn,replace_name)
                  # now continue on to add in the new block
                  if parent.lower() == new_lowerbn:        #the new block's requested parent just got renamed!!
                      parent = replace_name
               else:
                  raise StarError( "Attempt to replace existing block " + blockname)
            else:
               del self[new_lowerbn]
        self.dictionary.update({new_lowerbn:blockcontents})
        self.lower_keys.add(new_lowerbn)
        if parent is None:
           self.child_table[new_lowerbn]=self.PC(newblockname,None)
           self.visible_keys.append(new_lowerbn)
        else:
           if parent.lower() in self.lower_keys:
              if self.scoping == 'instance':
                 self.child_table[new_lowerbn]=self.PC(newblockname,parent.lower())
              else:
                 self.child_table[new_lowerbn]=self.PC(newblockname,parent.lower())
                 self.visible_keys.append(new_lowerbn)
           else:
               print 'Warning:Parent block %s does not exist for child %s' % (parent,newblockname)  
        return new_lowerbn  #in case calling routine wants to know

    def _rekey(self,oldname,newname,block_id=''):
        """The block with key [[oldname]] gets [[newname]] as a new key, but the printed name
           does not change unless [[block_id]] is given.  Prefer [[rename]] for a safe version."""
        move_block = self[oldname]    #old block
        is_visible = oldname in self.visible_keys
        move_block_info = self.child_table[oldname]    #old info
        move_block_children = [a for a in self.child_table.items() if a[1].parent==oldname]
        # now rewrite the necessary bits
        self.child_table.update(dict([(a[0],self.PC(a[1].block_id,newname)) for a in move_block_children]))
        del self[oldname]   #do this after updating child table so we don't delete children
        self.dictionary.update({newname:move_block})
        self.lower_keys.add(newname)
        if block_id == '':
           self.child_table.update({newname:move_block_info})
        else:
           self.child_table.update({newname:self.PC(block_id,move_block_info.parent)})
        if is_visible: self.visible_keys += [newname]

    def rename(self,oldname,newname):
        """Rename datablock from [[oldname]] to [[newname]]. Both key and printed name are changed.  No
           conformance checks are conducted."""
        realoldname = oldname.lower()
        realnewname = newname.lower()
        if realnewname in self.lower_keys:
            raise StarError,'Cannot change blockname %s to %s as %s already present' % (oldname,newname,newname)
        if realoldname not in self.lower_keys:
            raise KeyError,'Cannot find old block %s' % realoldname
        self._rekey(realoldname,realnewname,block_id=newname)
        
    def merge_fast(self,new_bc,parent=None):
        """Do a fast merge"""
        if self.standard is None:
            mode = 'replace' 
        else:
            mode = 'strict'
        overlap_flag = not self.lower_keys.isdisjoint(new_bc.lower_keys)
        if overlap_flag and mode != 'replace':
            double_keys = self.lower_keys.intersection(new_bc.lower_keys)
            for dup_key in double_keys:
                  our_parent = self.child_table[dup_key].parent
                  their_parent = new_bc.child_table[dup_key].parent
                  if (our_parent is None and their_parent is not None and parent is None) or\
                      parent is not None:  #rename our block
                    start_key = dup_key
                    while start_key in self.lower_keys: start_key = start_key+'+'
                    self._rekey(dup_key,start_key)
                    if parent.lower() == dup_key:  #we just renamed the prospective parent!
                        parent = start_key
                  elif our_parent is not None and their_parent is None and parent is None:
                    start_key = dup_key
                    while start_key in new_bc.lower_keys: start_key = start_key+'+'
                    new_bc._rekey(dup_key,start_key)
                  else: 
                    raise StarError("In strict merge mode:duplicate keys %s" % dup_key)
        self.dictionary.update(new_bc.dictionary) 
        self.lower_keys.update(new_bc.lower_keys)
        self.visible_keys += (list(new_bc.lower_keys))
        self.child_table.update(new_bc.child_table)
        if parent is not None:     #redo the child_table entries
              reparent_list = [(a[0],a[1].block_id) for a in new_bc.child_table.items() if a[1].parent==None]
              reparent_dict = [(a[0],self.PC(a[1],parent.lower())) for a in reparent_list]
              self.child_table.update(dict(reparent_dict))

    def merge(self,new_bc,mode=None,parent=None,single_block=[],
                   idblock="",match_att=[],match_function=None):
        if mode is None:
            if self.standard is None:
               mode = 'replace'
            else:
               mode = 'strict'
        if single_block:
            self[single_block[0]].merge(new_bc[single_block[1]],mode,
                                                   match_att=match_att,
                                                   match_function=match_function)
            return None
        base_keys = [a[1].block_id for a in self.child_table.items()]
        block_to_item = base_keys   #default
        new_keys = [a[1].block_id for a in new_bc.child_table.items()]    #get list of incoming blocks
        if match_att:
            #make a blockname -> item name map
            if match_function:
                block_to_item = map(lambda a:match_function(self[a]),self.keys())
            else:
                block_to_item = map(lambda a:self[a].get(match_att[0],None),self.keys())
            #print `block_to_item`
        for key in new_keys:        #run over incoming blocknames
            if key == idblock: continue    #skip dictionary id
            basekey = key           #default value
            if len(match_att)>0:
               attval = new_bc[key].get(match_att[0],0)  #0 if ignoring matching
            else:
               attval = 0
            for ii in range(len(block_to_item)):  #do this way to get looped names
                thisatt = block_to_item[ii]       #keyname in old block
                #print "Looking for %s in %s" % (attval,thisatt)
                if attval == thisatt or \
                   (isinstance(thisatt,list) and attval in thisatt):
                      basekey = base_keys.pop(ii)
                      block_to_item.remove(thisatt)
                      break
            if not self.has_key(basekey) or mode=="replace":
                new_parent = new_bc.get_parent(key)
                if parent is not None and new_parent is None:
                   new_parent = parent
                self.NewBlock(basekey,new_bc[key],parent=new_parent)   #add the block
            else:
                if mode=="strict":
                    raise StarError( "In strict merge mode: block %s in old and block %s in new files" % (basekey,key))
                elif mode=="overlay":
                    # print "Merging block %s with %s" % (basekey,key)
                    self[basekey].merge(new_bc[key],mode,match_att=match_att)
                else:  
                    raise StarError( "Merge called with unknown mode %s" % mode)
         
    def checknamelengths(self,target_block,maxlength=-1):
        if maxlength < 0:
            return
        else:
            toolong = filter(lambda a:len(a)>maxlength, target_block.keys())
        outstring = ""
        for it in toolong: outstring += "\n" + it
        if toolong:
           raise StarError( 'Following data names too long:' + outstring)

    def get_all(self,item_name):
        raw_values = map(lambda a:self[a].get(item_name),self.keys())
        raw_values = filter(lambda a:a != None, raw_values)
        ret_vals = []
        for rv in raw_values:
            if isinstance(rv,list):
                for rvv in rv:
                    if rvv not in ret_vals: ret_vals.append(rvv)
            else:
                if rv not in ret_vals: ret_vals.append(rv)
        return ret_vals

    def __setattr__(self,attr_name,newval):
        if attr_name == 'scoping':
            if newval not in ('dictionary','instance'):
                raise StarError("Star file may only have 'dictionary' or 'instance' scoping, not %s" % newval)
            if newval == 'dictionary':
                self.visible_keys = [a for a in self.lower_keys] 
            else:
                #only top-level datablocks visible
                self.visible_keys = [a[0] for a in self.child_table.items() if a[1].parent==None]
        object.__setattr__(self,attr_name,newval)

    def get_parent(self,blockname):
        """Return the name of the block enclosing [[blockname]] in canonical form (lower case)"""
        possibles = (a for a in self.child_table.items() if a[0] == blockname.lower())
        try:
            first = possibles.next()   #get first one
        except:
            raise StarError('no parent for %s' % blockname)
        try:
           second = possibles.next()
        except StopIteration:
           return first[1].parent
        raise StarError('More than one parent for %s' % blockname)

    def get_roots(self):
        """Get the top-level blocks"""
        return [a for a in self.child_table.items() if a[1].parent==None]

    def get_children(self,blockname,include_parent=False,scoping='dictionary'):
        """Get all children of [[blockname]] as a block collection. If [[include_parent]] is
        True, the parent block will also be included in the block collection as the root."""
        newbc = BlockCollection()
        block_lower = blockname.lower()
        proto_child_table = [a for a in self.child_table.items() if self.is_child_of_parent(block_lower,a[1].block_id)]
        newbc.child_table = dict(proto_child_table)
        if not include_parent:
           newbc.child_table.update(dict([(a[0],self.PC(a[1].block_id,None)) for a in proto_child_table if a[1].parent == block_lower]))
        newbc.lower_keys = set([a[0] for a in proto_child_table])
        newbc.dictionary = dict((a[0],self.dictionary[a[0]]) for a in proto_child_table)
        if include_parent:
            newbc.child_table.update({block_lower:self.PC(self.child_table[block_lower].block_id,None)})
            newbc.lower_keys.add(block_lower)
            newbc.dictionary.update({block_lower:self.dictionary[block_lower]})
        newbc.scoping = scoping
        return newbc

    def get_immediate_children(self,parentname):
        """Get the next level of children of the given block as a list, without nested levels"""
        child_handles = [a for a in self.child_table.items() if a[1].parent == parentname.lower()]
        return child_handles

    def get_child_list(self,parentname):
        """Get a list of all child categories"""
        child_handles = [a[0] for a in self.child_table.items() if self.is_child_of_parent(parentname.lower(),a[0])]
        return child_handles

    def is_child_of_parent(self,parentname,blockname):
        """Recursively search for children of blockname, case is important for now"""
        checkname = parentname.lower()
        more_children = [a[0] for a in self.child_table.items() if a[1].parent == checkname]
        if blockname.lower() in more_children:
           return True
        else:
           for one_child in more_children:
               if self.is_child_of_parent(one_child,blockname): return True
        return False
           
    def set_parent(self,parentname,childname):
        """Set the parent block"""
        # first check that both blocks exist
        if parentname.lower() not in self.lower_keys:
            raise KeyError('Parent block %s does not exist' % parentname)
        if childname.lower() not in self.lower_keys:
            raise KeyError('Child block %s does not exist' % childname)
        old_entry = self.child_table[childname.lower()]
        self.child_table[childname.lower()]=self.PC(old_entry.block_id,
               parentname.lower())
        self.scoping = self.scoping #reset visibility
            
    def WriteOut(self,comment='',wraplength=80,maxoutlength=2048):
        import cStringIO
        if not comment:
            comment = self.header_comment
        outstring = cStringIO.StringIO()
        outstring.write(comment)
        # loop over top-level
        top_block_names = [(a[0],a[1].block_id) for a in self.child_table.items() if a[1].parent is None]
        for blockref,blockname in top_block_names:
            outstring.write('\n' + 'data_' +blockname+'\n')
            child_names = [(a[0],a[1].block_id) for a in self.child_table.items() if a[1].parent==blockref]
            if self.standard == 'Dic':              #put contents before save frames
              self[blockref].SetOutputLength(wraplength,maxoutlength)
              outstring.write(str(self[blockref]))
            for child_ref,child_name in child_names:
                outstring.write('\n' + 'save_' + child_name + '\n')
                self.block_to_string(child_ref,child_name,outstring,4)    
                outstring.write('\n' + 'save_'+ '\n')   
            if self.standard != 'Dic':              #put contents after save frames
                self[blockref].SetOutputLength(wraplength,maxoutlength)
                outstring.write(str(self[blockref]))
        returnstring =  outstring.getvalue()
        outstring.close()
        return returnstring

    def block_to_string(self,block_ref,block_id,outstring,indentlevel=0):
        """Output a complete datablock indexed by [[block_ref]] and named [[block_id]], including children"""
        child_names = [(a[0],a[1].block_id) for a in self.child_table.items() if a[1].parent==block_ref]
        if self.standard == 'Dic':
            outstring.write(str(self[block_ref]))
        for child_ref,child_name in child_names:
            outstring.write('\n' + 'save_' + child_name + '\n')
            self.block_to_string(child_ref,child_name,outstring,indentlevel)
            outstring.write('\n' + '  '*indentlevel + 'save_' + '\n')
        if self.standard != 'Dic':
            outstring.write(str(self[block_ref]))
        

class StarFile(BlockCollection):
    def __init__(self,datasource=None,maxinlength=-1,maxoutlength=0,
                scoping='instance',grammar='1.1',scantype='standard',
                **kwargs):
        super(StarFile,self).__init__(datasource=datasource,**kwargs)
        self.my_uri = getattr(datasource,'my_uri','')
        if maxoutlength == 0:
            self.maxoutlength = 2048 
        else:
            self.maxoutlength = maxoutlength
        self.scoping = scoping
        if isinstance(datasource,basestring) or hasattr(datasource,"read"):
            ReadStar(datasource,prepared=self,grammar=grammar,scantype=scantype,
            maxlength = maxinlength)
        self.header_comment = \
"""#\\#STAR
##########################################################################
#               STAR Format file 
#               Produced by PySTARRW module
# 
#  This is a STAR file.  STAR is a superset of the CIF file type.  For
#  more information, please refer to International Tables for Crystallography,
#  Volume G, Chapter 2.1
#
##########################################################################
"""
    def set_uri(self,my_uri): self.my_uri = my_uri


from StringIO import StringIO
import math
class CIFStringIO(StringIO):
    def __init__(self,target_width=80,**kwargs):
        StringIO.__init__(self,**kwargs)
        self.currentpos = 0
        self.target_width = target_width
        self.tabwidth = -1
        self.indentlist = [0]

    def write(self,outstring,canbreak=False,mustbreak=False,do_tab=True,newindent=False,unindent=False,startcol=-1):
        """Write a string with correct linebreak, tabs and indents"""
        # do we need to break?
        if mustbreak:    #insert a new line and indent
            StringIO.write(self,'\n' + ' '*self.indentlist[-1])
            self.currentpos = self.indentlist[-1]
        if self.currentpos+len(outstring)>self.target_width: #try to break
            if canbreak:
                StringIO.write(self,'\n'+' '*self.indentlist[-1])
                self.currentpos = self.indentlist[-1]
        if newindent:           #indent by current amount
            if self.indentlist[-1] == 0:    #first time
                self.indentlist.append(self.currentpos)
                print 'Indentlist: ' + `self.indentlist`
            else:
                self.indentlist.append(self.indentlist[-1]+2)
        elif unindent:
            if len(self.indentlist)>1:
                self.indentlist.pop()
            else:
                print 'Warning: cannot unindent any further'
        #handle tabs
        if self.tabwidth >0 and do_tab:
            next_stop = ((self.currentpos//self.tabwidth)+1)*self.tabwidth
            #print 'Currentpos %d: Next tab stop at %d' % (self.currentpos,next_stop)
            if self.currentpos < next_stop:
                StringIO.write(self,(next_stop-self.currentpos)*' ')
                self.currentpos = next_stop
        #now output the string
        StringIO.write(self,outstring)
        last_line_break = outstring.rfind('\n')
        if last_line_break >=0:
            self.currentpos = len(outstring)-last_line_break
        else:
            self.currentpos = self.currentpos + len(outstring)
        
    def set_tab(self,tabwidth):
        """Set the tab stop position"""
        self.tabwidth = tabwidth

class StarError(Exception):
    def __init__(self,value):
        self.value = value
    def __str__(self):
        return '\nStar Format error: '+ self.value 

class StarLengthError(Exception):
    def __init__(self,value):
        self.value = value
    def __str__(self):
        return '\nStar length error: ' + self.value

def ReadStar(filename,prepared = StarFile(),maxlength=-1,
             scantype='standard',grammar='1.1',CBF=False):
    import string
    import codecs
    # save desired scoping
    save_scoping = prepared.scoping
    if grammar=="1.1":
        import YappsStarParser_1_1 as Y
    elif grammar=="1.0":
        import YappsStarParser_1_0 as Y
    elif grammar=="DDLm":
        import YappsStarParser_DDLm as Y
    if isinstance(filename,basestring):
        filestream = urlopen(filename)
    else:
        filestream = filename   #already opened for us
    my_uri = ""
    if hasattr(filestream,"geturl"): 
        my_uri = filestream.geturl()
    text = unicode(filestream.read(),"utf8")
    if isinstance(filename,basestring): #we opened it, we close it
        filestream.close()
    if not text:      # empty file, return empty block
        return StarFile().set_uri(my_uri)
    # filter out non-ASCII characters in CBF files if required.  We assume
    # that the binary is enclosed in a fixed string that occurs
    # nowhere else.
    if CBF:
       text_bits  = text.split("-BINARY-FORMAT-SECTION-") 
       text = text_bits[0] 
       for section in range(2,len(text_bits),2):
           text = text+" (binary omitted)"+text_bits[section]
    # we recognise ctrl-Z as end of file
    endoffile = text.find('\x1a')
    if endoffile >= 0: 
        text = text[:endoffile]
    split = string.split(text,'\n')
    if maxlength > 0:
        toolong = filter(lambda a:len(a)>maxlength,split)
        if toolong:
            pos = split.index(toolong[0])
            raise StarError( 'Line %d contains more than %d characters' % (pos+1,maxlength))
    if scantype == 'standard':
            parser = Y.StarParser(Y.StarParserScanner(text))
    else:
            parser = Y.StarParser(Y.yappsrt.Scanner(None,[],text,scantype='flex'))
    proto_star = None
    try:
        proto_star = getattr(parser,"input")(prepared)
    except Y.yappsrt.SyntaxError,e:
           input = parser._scanner.input
           Y.yappsrt.print_error(input, e, parser._scanner)
    except Y.yappsrt.NoMoreTokens:
           print >>sys.stderr, 'Could not complete parsing; stopped around here:'
           print >>sys.stderr, parser._scanner
    except ValueError:
           pass
    if proto_star == None:
        errorstring = 'Syntax error in input file: last value parsed was %s' % Y.lastval
        errorstring = errorstring + '\nParser status: %s' % `parser._scanner`
        raise StarError( errorstring)
    # set visibility correctly
    proto_star.scoping = 'dictionary'
    proto_star.set_uri(my_uri)
    proto_star.scoping = save_scoping
    return proto_star

def get_dim(dataitem,current=0,packlen=0):
    zerotypes = [int, long, float, basestring]
    if type(dataitem) in zerotypes:
        return current, packlen
    if not dataitem.__class__ == ().__class__ and \
       not dataitem.__class__ == [].__class__:
       return current, packlen
    elif len(dataitem)>0: 
    #    print "Get_dim: %d: %s" % (current,`dataitem`)
        return get_dim(dataitem[0],current+1,len(dataitem))
    else: return current+1,0
    
def apply_line_folding(instring,minwraplength=60,maxwraplength=80):
    """Insert line folding characters into instring between min/max wraplength"""
    # first check that we need to do this
    lines = instring.split('\n')
    line_len = [len(l) for l in lines]
    if max(line_len) < maxwraplength and re.match("\\[ \v\t\f]*\n",instring) is None:
        return instring
    outstring = "\\\n"   #header
    for l in lines:
        if len(l) < maxwraplength:
            outstring = outstring + l
            if len(l) > 0 and l[-1]=='\\': #who'da thunk it?  A line ending with a backslash
                    outstring = outstring + "\\\n"  #
            outstring = outstring + "\n"  #  put back the split character
        else:
            current_bit = l
            while len(current_bit) > maxwraplength:
                space_pos = re.search('[ \v\f\t]+',current_bit[minwraplength:])
                if space_pos is not None and space_pos.start()[^;\\\n][^\n\\\\]+)(?P\\\\{1,2}[ \t\v\f]*\n)",instring)
    if prefix_match is not None:
        prefix_text = prefix_match.group('prefix')
        print 'Found prefix %s' % prefix_text
        prefix_end = prefix_match.end('folding')
        # keep any line folding instructions
        if prefix_match.group('folding')[:2]=='\\\\':  #two backslashes
            outstring = instring[prefix_match.end('folding')-1:].replace("\n"+prefix_text,"\n")
            return "\\" + outstring  #keep line folding first line
        else:
            outstring = instring[prefix_match.end('folding')-1:].replace("\n"+prefix_text,"\n")
            return outstring[1:]   #drop first line ending, no longer necessary
    else:
        return instring
    

def listify(item):
    if isinstance(item,basestring): return [item]
    else: return item

#Transpose the list of lists passed to us
def transpose(base_list):
    new_lofl = []
    full_length = len(base_list)
    opt_range = range(full_length)
    for i in range(len(base_list[0])):
       new_packet = [] 
       for j in opt_range:
          new_packet.append(base_list[j][i])
       new_lofl.append(new_packet)
    return new_lofl


def check_stringiness(data):
   """Check that the contents of data are all strings"""
   if not hasattr(data,'dtype'):   #so not Numpy
       from numbers import Number
       if isinstance(data,Number): return False
       elif isinstance(data,basestring): return True
       elif data is None:return False  #should be data are None :)
       else:
           for one_item in data:
               if not check_stringiness(one_item): return False
           return True   #all must be strings
   else:   #numerical python
       import numpy
       if data.ndim == 0:    #a bare value
           if data.dtype.kind in ['S','U']: return True
           else: return False
       else:
           for one_item in numpy.nditer(data):
               print 'numpy data: ' + `one_item`
               if not check_stringiness(one_item): return False
           return True


#No documentation flags

Functions

def ReadStar(

filename, prepared=<__pdoc_file_module__.StarFile object at 0x4094b70c>, maxlength=-1, scantype='standard', grammar='1.1', CBF=False)

def ReadStar(filename,prepared = StarFile(),maxlength=-1,
             scantype='standard',grammar='1.1',CBF=False):
    import string
    import codecs
    # save desired scoping
    save_scoping = prepared.scoping
    if grammar=="1.1":
        import YappsStarParser_1_1 as Y
    elif grammar=="1.0":
        import YappsStarParser_1_0 as Y
    elif grammar=="DDLm":
        import YappsStarParser_DDLm as Y
    if isinstance(filename,basestring):
        filestream = urlopen(filename)
    else:
        filestream = filename   #already opened for us
    my_uri = ""
    if hasattr(filestream,"geturl"): 
        my_uri = filestream.geturl()
    text = unicode(filestream.read(),"utf8")
    if isinstance(filename,basestring): #we opened it, we close it
        filestream.close()
    if not text:      # empty file, return empty block
        return StarFile().set_uri(my_uri)
    # filter out non-ASCII characters in CBF files if required.  We assume
    # that the binary is enclosed in a fixed string that occurs
    # nowhere else.
    if CBF:
       text_bits  = text.split("-BINARY-FORMAT-SECTION-") 
       text = text_bits[0] 
       for section in range(2,len(text_bits),2):
           text = text+" (binary omitted)"+text_bits[section]
    # we recognise ctrl-Z as end of file
    endoffile = text.find('\x1a')
    if endoffile >= 0: 
        text = text[:endoffile]
    split = string.split(text,'\n')
    if maxlength > 0:
        toolong = filter(lambda a:len(a)>maxlength,split)
        if toolong:
            pos = split.index(toolong[0])
            raise StarError( 'Line %d contains more than %d characters' % (pos+1,maxlength))
    if scantype == 'standard':
            parser = Y.StarParser(Y.StarParserScanner(text))
    else:
            parser = Y.StarParser(Y.yappsrt.Scanner(None,[],text,scantype='flex'))
    proto_star = None
    try:
        proto_star = getattr(parser,"input")(prepared)
    except Y.yappsrt.SyntaxError,e:
           input = parser._scanner.input
           Y.yappsrt.print_error(input, e, parser._scanner)
    except Y.yappsrt.NoMoreTokens:
           print >>sys.stderr, 'Could not complete parsing; stopped around here:'
           print >>sys.stderr, parser._scanner
    except ValueError:
           pass
    if proto_star == None:
        errorstring = 'Syntax error in input file: last value parsed was %s' % Y.lastval
        errorstring = errorstring + '\nParser status: %s' % `parser._scanner`
        raise StarError( errorstring)
    # set visibility correctly
    proto_star.scoping = 'dictionary'
    proto_star.set_uri(my_uri)
    proto_star.scoping = save_scoping
    return proto_star

def apply_line_folding(

instring, minwraplength=60, maxwraplength=80)

Insert line folding characters into instring between min/max wraplength

def apply_line_folding(instring,minwraplength=60,maxwraplength=80):
    """Insert line folding characters into instring between min/max wraplength"""
    # first check that we need to do this
    lines = instring.split('\n')
    line_len = [len(l) for l in lines]
    if max(line_len) < maxwraplength and re.match("\\[ \v\t\f]*\n",instring) is None:
        return instring
    outstring = "\\\n"   #header
    for l in lines:
        if len(l) < maxwraplength:
            outstring = outstring + l
            if len(l) > 0 and l[-1]=='\\': #who'da thunk it?  A line ending with a backslash
                    outstring = outstring + "\\\n"  #
            outstring = outstring + "\n"  #  put back the split character
        else:
            current_bit = l
            while len(current_bit) > maxwraplength:
                space_pos = re.search('[ \v\f\t]+',current_bit[minwraplength:])
                if space_pos is not None and space_pos.start()

def apply_line_prefix(

instring, prefix)

Prefix every line in instring with prefix

def apply_line_prefix(instring,prefix):
    """Prefix every line in instring with prefix"""
    if prefix[0] != ";" and "\\" not in prefix:
        header = re.match(r"(\\[ \v\t\f]*" +"\n)",instring)
        if header is not None:
            print 'Found line folded string for prefixing...'
            not_header = instring[header.end():]
            outstring = prefix + "\\\\\n" + prefix
        else:
            print 'No folding in input string...'
            not_header = instring
            outstring = prefix + "\\\n" + prefix
        outstring = outstring + not_header.replace("\n","\n"+prefix)
        return outstring
    raise StarError, "Requested prefix starts with semicolon or contains a backslash: " + prefix

def check_stringiness(

data)

Check that the contents of data are all strings

def check_stringiness(data):
   """Check that the contents of data are all strings"""
   if not hasattr(data,'dtype'):   #so not Numpy
       from numbers import Number
       if isinstance(data,Number): return False
       elif isinstance(data,basestring): return True
       elif data is None:return False  #should be data are None :)
       else:
           for one_item in data:
               if not check_stringiness(one_item): return False
           return True   #all must be strings
   else:   #numerical python
       import numpy
       if data.ndim == 0:    #a bare value
           if data.dtype.kind in ['S','U']: return True
           else: return False
       else:
           for one_item in numpy.nditer(data):
               print 'numpy data: ' + `one_item`
               if not check_stringiness(one_item): return False
           return True

def get_dim(

dataitem, current=0, packlen=0)

def get_dim(dataitem,current=0,packlen=0):
    zerotypes = [int, long, float, basestring]
    if type(dataitem) in zerotypes:
        return current, packlen
    if not dataitem.__class__ == ().__class__ and \
       not dataitem.__class__ == [].__class__:
       return current, packlen
    elif len(dataitem)>0: 
    #    print "Get_dim: %d: %s" % (current,`dataitem`)
        return get_dim(dataitem[0],current+1,len(dataitem))
    else: return current+1,0

def listify(

item)

def listify(item):
    if isinstance(item,basestring): return [item]
    else: return item

def remove_line_folding(

instring)

Remove line folding from instring

def remove_line_folding(instring):
    """Remove line folding from instring"""
    if re.match(r"\\[ \v\t\f]*" +"\n",instring) is not None:
        return re.sub(r"\\[ \v\t\f]*$" + "\n?","",instring,flags=re.M)
    else:
        return instring

def remove_line_prefix(

instring)

Remove prefix from every line if present

def remove_line_prefix(instring):
    """Remove prefix from every line if present"""
    prefix_match = re.match("(?P[^;\\\n][^\n\\\\]+)(?P\\\\{1,2}[ \t\v\f]*\n)",instring)
    if prefix_match is not None:
        prefix_text = prefix_match.group('prefix')
        print 'Found prefix %s' % prefix_text
        prefix_end = prefix_match.end('folding')
        # keep any line folding instructions
        if prefix_match.group('folding')[:2]=='\\\\':  #two backslashes
            outstring = instring[prefix_match.end('folding')-1:].replace("\n"+prefix_text,"\n")
            return "\\" + outstring  #keep line folding first line
        else:
            outstring = instring[prefix_match.end('folding')-1:].replace("\n"+prefix_text,"\n")
            return outstring[1:]   #drop first line ending, no longer necessary
    else:
        return instring

def transpose(

base_list)

def transpose(base_list):
    new_lofl = []
    full_length = len(base_list)
    opt_range = range(full_length)
    for i in range(len(base_list[0])):
       new_packet = [] 
       for j in opt_range:
          new_packet.append(base_list[j][i])
       new_lofl.append(new_packet)
    return new_lofl

Classes

class BlockCollection

class BlockCollection(object):
    def __init__(self,datasource=None,standard='CIF',blocktype = StarBlock,
                 characterset='ascii',scoping='instance',parent_id=None,**kwargs):
        import collections
        self.dictionary = {}
        self.standard = standard
        self.lower_keys = set()           # short_cuts
        self.renamed = {}
        self.characterset = characterset
        self.PC = collections.namedtuple('PC',['block_id','parent'])
        self.child_table = {}
        self.visible_keys = []            # for efficiency
        self.parent_id = parent_id
        self.scoping = scoping  #will trigger setting of child table
        self.blocktype = blocktype
        if isinstance(datasource,BlockCollection):
            self.merge_fast(datasource)
            self.scoping = scoping   #reset visibility
        elif isinstance(datasource,dict):
            for key,value in datasource.items():
                 self[key]= value
        self.header_comment = ''
     
    def unlock(self):
        """Allow overwriting of all blocks in this collection"""
        for a in self.lower_keys:
            self[a].overwrite=True

    def lock(self):
        """Disallow overwriting for all blocks in this collection"""
        for a in self.lower_keys:
            self[a].overwrite = False

    def __str__(self):
        return self.WriteOut()

    def __setitem__(self,key,value):
        self.NewBlock(key,value,parent=None)

    def __getitem__(self,key):
        if isinstance(key,basestring):
           lowerkey = key.lower()
           if lowerkey in self.lower_keys:
               return self.dictionary[lowerkey]
           #print 'Visible keys:' + `self.visible_keys`
           #print 'All keys' + `self.lower_keys`
           #print 'Child table' + `self.child_table`
           raise KeyError,'No such item %s' % key

    # we have to get an ordered list of the current keys,
    # as we'll have to delete one of them anyway.
    # Deletion will delete any key regardless of visibility

    def __delitem__(self,key):
        dummy = self[key]   #raise error if not present
        lowerkey = key.lower()
        # get rid of all children recursively as well
        children = [a[0] for a in self.child_table.items() if a[1].parent == lowerkey]
        for child in children:
            del self[child]   #recursive call
        del self.dictionary[lowerkey]
        del self.child_table[lowerkey]
        try:
            self.visible_keys.remove(lowerkey)
        except KeyError:
            pass
        self.lower_keys.remove(lowerkey)

    def __len__(self):
        return len(self.visible_keys)

    def __contains__(self,item):
        """Support the 'in' operator"""
        return self.has_key(item)

    # We iterate over all visible
    def __iter__(self):
        for one_block in self.keys():
            yield self[one_block]

    # TODO: handle different case
    def keys(self):
        return self.visible_keys

    # changes to take case independence into account
    def has_key(self,key):
        if not isinstance(key,basestring): return 0
        if key.lower() in self.visible_keys:
            return 1
        return 0

    def get(self,key,default=None):
        if self.has_key(key):     # take account of case
            return self.__getitem__(key)
        else:
            return default

    def clear(self):
        self.dictionary.clear()
        self.lower_keys = set()
        self.child_table = {}
        self.visible_keys = []

    def copy(self):   
        newcopy = self.dictionary.copy()  #all blocks
        newcopy = BlockCollection('',newcopy,parent_id=self.parent_id)
        newcopy.child_table = self.child_table.copy()
        newcopy.lower_keys = self.lower_keys
        newcopy.characterset = self.characterset
        newcopy.scoping = self.scoping  #this sets visible keys
        return newcopy

    def update(self,adict):
        for key in adict.keys():
            self[key] = adict[key]

    def items(self):
        return [(a,self[a]) for a in self.keys()]

    def first_block(self):
        """Return the 'first' block.  This is not necessarily the first block in the file."""
        if self.keys():
            return self[self.keys()[0]]

    def NewBlock(self,blockname,blockcontents=None,fix=True,parent=None):
        if blockcontents is None:
            blockcontents = StarBlock()
        if self.standard == "CIF":
            blockcontents.setmaxnamelength(75)
        if len(blockname)>75:
                 raise StarError , 'Blockname %s is longer than 75 characters' % blockname
        if fix:
            newblockname = re.sub('[  \t]','_',blockname)
        else: newblockname = blockname
        new_lowerbn = newblockname.lower()
        if new_lowerbn in self.lower_keys:
            if self.standard is not None:    #already there
               toplevelnames = [a[0] for a in self.child_table.items() if a[1].parent==None]
               if parent is None and new_lowerbn not in toplevelnames:  #can give a new key to this one
                  while new_lowerbn in self.lower_keys: new_lowerbn = new_lowerbn + '+'
               elif parent is not None and new_lowerbn in toplevelnames: #can fix a different one
                  replace_name = new_lowerbn            
                  while replace_name in self.lower_keys: replace_name = replace_name + '+'
                  self._rekey(new_lowerbn,replace_name)
                  # now continue on to add in the new block
                  if parent.lower() == new_lowerbn:        #the new block's requested parent just got renamed!!
                      parent = replace_name
               else:
                  raise StarError( "Attempt to replace existing block " + blockname)
            else:
               del self[new_lowerbn]
        self.dictionary.update({new_lowerbn:blockcontents})
        self.lower_keys.add(new_lowerbn)
        if parent is None:
           self.child_table[new_lowerbn]=self.PC(newblockname,None)
           self.visible_keys.append(new_lowerbn)
        else:
           if parent.lower() in self.lower_keys:
              if self.scoping == 'instance':
                 self.child_table[new_lowerbn]=self.PC(newblockname,parent.lower())
              else:
                 self.child_table[new_lowerbn]=self.PC(newblockname,parent.lower())
                 self.visible_keys.append(new_lowerbn)
           else:
               print 'Warning:Parent block %s does not exist for child %s' % (parent,newblockname)  
        return new_lowerbn  #in case calling routine wants to know

    def _rekey(self,oldname,newname,block_id=''):
        """The block with key [[oldname]] gets [[newname]] as a new key, but the printed name
           does not change unless [[block_id]] is given.  Prefer [[rename]] for a safe version."""
        move_block = self[oldname]    #old block
        is_visible = oldname in self.visible_keys
        move_block_info = self.child_table[oldname]    #old info
        move_block_children = [a for a in self.child_table.items() if a[1].parent==oldname]
        # now rewrite the necessary bits
        self.child_table.update(dict([(a[0],self.PC(a[1].block_id,newname)) for a in move_block_children]))
        del self[oldname]   #do this after updating child table so we don't delete children
        self.dictionary.update({newname:move_block})
        self.lower_keys.add(newname)
        if block_id == '':
           self.child_table.update({newname:move_block_info})
        else:
           self.child_table.update({newname:self.PC(block_id,move_block_info.parent)})
        if is_visible: self.visible_keys += [newname]

    def rename(self,oldname,newname):
        """Rename datablock from [[oldname]] to [[newname]]. Both key and printed name are changed.  No
           conformance checks are conducted."""
        realoldname = oldname.lower()
        realnewname = newname.lower()
        if realnewname in self.lower_keys:
            raise StarError,'Cannot change blockname %s to %s as %s already present' % (oldname,newname,newname)
        if realoldname not in self.lower_keys:
            raise KeyError,'Cannot find old block %s' % realoldname
        self._rekey(realoldname,realnewname,block_id=newname)
        
    def merge_fast(self,new_bc,parent=None):
        """Do a fast merge"""
        if self.standard is None:
            mode = 'replace' 
        else:
            mode = 'strict'
        overlap_flag = not self.lower_keys.isdisjoint(new_bc.lower_keys)
        if overlap_flag and mode != 'replace':
            double_keys = self.lower_keys.intersection(new_bc.lower_keys)
            for dup_key in double_keys:
                  our_parent = self.child_table[dup_key].parent
                  their_parent = new_bc.child_table[dup_key].parent
                  if (our_parent is None and their_parent is not None and parent is None) or\
                      parent is not None:  #rename our block
                    start_key = dup_key
                    while start_key in self.lower_keys: start_key = start_key+'+'
                    self._rekey(dup_key,start_key)
                    if parent.lower() == dup_key:  #we just renamed the prospective parent!
                        parent = start_key
                  elif our_parent is not None and their_parent is None and parent is None:
                    start_key = dup_key
                    while start_key in new_bc.lower_keys: start_key = start_key+'+'
                    new_bc._rekey(dup_key,start_key)
                  else: 
                    raise StarError("In strict merge mode:duplicate keys %s" % dup_key)
        self.dictionary.update(new_bc.dictionary) 
        self.lower_keys.update(new_bc.lower_keys)
        self.visible_keys += (list(new_bc.lower_keys))
        self.child_table.update(new_bc.child_table)
        if parent is not None:     #redo the child_table entries
              reparent_list = [(a[0],a[1].block_id) for a in new_bc.child_table.items() if a[1].parent==None]
              reparent_dict = [(a[0],self.PC(a[1],parent.lower())) for a in reparent_list]
              self.child_table.update(dict(reparent_dict))

    def merge(self,new_bc,mode=None,parent=None,single_block=[],
                   idblock="",match_att=[],match_function=None):
        if mode is None:
            if self.standard is None:
               mode = 'replace'
            else:
               mode = 'strict'
        if single_block:
            self[single_block[0]].merge(new_bc[single_block[1]],mode,
                                                   match_att=match_att,
                                                   match_function=match_function)
            return None
        base_keys = [a[1].block_id for a in self.child_table.items()]
        block_to_item = base_keys   #default
        new_keys = [a[1].block_id for a in new_bc.child_table.items()]    #get list of incoming blocks
        if match_att:
            #make a blockname -> item name map
            if match_function:
                block_to_item = map(lambda a:match_function(self[a]),self.keys())
            else:
                block_to_item = map(lambda a:self[a].get(match_att[0],None),self.keys())
            #print `block_to_item`
        for key in new_keys:        #run over incoming blocknames
            if key == idblock: continue    #skip dictionary id
            basekey = key           #default value
            if len(match_att)>0:
               attval = new_bc[key].get(match_att[0],0)  #0 if ignoring matching
            else:
               attval = 0
            for ii in range(len(block_to_item)):  #do this way to get looped names
                thisatt = block_to_item[ii]       #keyname in old block
                #print "Looking for %s in %s" % (attval,thisatt)
                if attval == thisatt or \
                   (isinstance(thisatt,list) and attval in thisatt):
                      basekey = base_keys.pop(ii)
                      block_to_item.remove(thisatt)
                      break
            if not self.has_key(basekey) or mode=="replace":
                new_parent = new_bc.get_parent(key)
                if parent is not None and new_parent is None:
                   new_parent = parent
                self.NewBlock(basekey,new_bc[key],parent=new_parent)   #add the block
            else:
                if mode=="strict":
                    raise StarError( "In strict merge mode: block %s in old and block %s in new files" % (basekey,key))
                elif mode=="overlay":
                    # print "Merging block %s with %s" % (basekey,key)
                    self[basekey].merge(new_bc[key],mode,match_att=match_att)
                else:  
                    raise StarError( "Merge called with unknown mode %s" % mode)
         
    def checknamelengths(self,target_block,maxlength=-1):
        if maxlength < 0:
            return
        else:
            toolong = filter(lambda a:len(a)>maxlength, target_block.keys())
        outstring = ""
        for it in toolong: outstring += "\n" + it
        if toolong:
           raise StarError( 'Following data names too long:' + outstring)

    def get_all(self,item_name):
        raw_values = map(lambda a:self[a].get(item_name),self.keys())
        raw_values = filter(lambda a:a != None, raw_values)
        ret_vals = []
        for rv in raw_values:
            if isinstance(rv,list):
                for rvv in rv:
                    if rvv not in ret_vals: ret_vals.append(rvv)
            else:
                if rv not in ret_vals: ret_vals.append(rv)
        return ret_vals

    def __setattr__(self,attr_name,newval):
        if attr_name == 'scoping':
            if newval not in ('dictionary','instance'):
                raise StarError("Star file may only have 'dictionary' or 'instance' scoping, not %s" % newval)
            if newval == 'dictionary':
                self.visible_keys = [a for a in self.lower_keys] 
            else:
                #only top-level datablocks visible
                self.visible_keys = [a[0] for a in self.child_table.items() if a[1].parent==None]
        object.__setattr__(self,attr_name,newval)

    def get_parent(self,blockname):
        """Return the name of the block enclosing [[blockname]] in canonical form (lower case)"""
        possibles = (a for a in self.child_table.items() if a[0] == blockname.lower())
        try:
            first = possibles.next()   #get first one
        except:
            raise StarError('no parent for %s' % blockname)
        try:
           second = possibles.next()
        except StopIteration:
           return first[1].parent
        raise StarError('More than one parent for %s' % blockname)

    def get_roots(self):
        """Get the top-level blocks"""
        return [a for a in self.child_table.items() if a[1].parent==None]

    def get_children(self,blockname,include_parent=False,scoping='dictionary'):
        """Get all children of [[blockname]] as a block collection. If [[include_parent]] is
        True, the parent block will also be included in the block collection as the root."""
        newbc = BlockCollection()
        block_lower = blockname.lower()
        proto_child_table = [a for a in self.child_table.items() if self.is_child_of_parent(block_lower,a[1].block_id)]
        newbc.child_table = dict(proto_child_table)
        if not include_parent:
           newbc.child_table.update(dict([(a[0],self.PC(a[1].block_id,None)) for a in proto_child_table if a[1].parent == block_lower]))
        newbc.lower_keys = set([a[0] for a in proto_child_table])
        newbc.dictionary = dict((a[0],self.dictionary[a[0]]) for a in proto_child_table)
        if include_parent:
            newbc.child_table.update({block_lower:self.PC(self.child_table[block_lower].block_id,None)})
            newbc.lower_keys.add(block_lower)
            newbc.dictionary.update({block_lower:self.dictionary[block_lower]})
        newbc.scoping = scoping
        return newbc

    def get_immediate_children(self,parentname):
        """Get the next level of children of the given block as a list, without nested levels"""
        child_handles = [a for a in self.child_table.items() if a[1].parent == parentname.lower()]
        return child_handles

    def get_child_list(self,parentname):
        """Get a list of all child categories"""
        child_handles = [a[0] for a in self.child_table.items() if self.is_child_of_parent(parentname.lower(),a[0])]
        return child_handles

    def is_child_of_parent(self,parentname,blockname):
        """Recursively search for children of blockname, case is important for now"""
        checkname = parentname.lower()
        more_children = [a[0] for a in self.child_table.items() if a[1].parent == checkname]
        if blockname.lower() in more_children:
           return True
        else:
           for one_child in more_children:
               if self.is_child_of_parent(one_child,blockname): return True
        return False
           
    def set_parent(self,parentname,childname):
        """Set the parent block"""
        # first check that both blocks exist
        if parentname.lower() not in self.lower_keys:
            raise KeyError('Parent block %s does not exist' % parentname)
        if childname.lower() not in self.lower_keys:
            raise KeyError('Child block %s does not exist' % childname)
        old_entry = self.child_table[childname.lower()]
        self.child_table[childname.lower()]=self.PC(old_entry.block_id,
               parentname.lower())
        self.scoping = self.scoping #reset visibility
            
    def WriteOut(self,comment='',wraplength=80,maxoutlength=2048):
        import cStringIO
        if not comment:
            comment = self.header_comment
        outstring = cStringIO.StringIO()
        outstring.write(comment)
        # loop over top-level
        top_block_names = [(a[0],a[1].block_id) for a in self.child_table.items() if a[1].parent is None]
        for blockref,blockname in top_block_names:
            outstring.write('\n' + 'data_' +blockname+'\n')
            child_names = [(a[0],a[1].block_id) for a in self.child_table.items() if a[1].parent==blockref]
            if self.standard == 'Dic':              #put contents before save frames
              self[blockref].SetOutputLength(wraplength,maxoutlength)
              outstring.write(str(self[blockref]))
            for child_ref,child_name in child_names:
                outstring.write('\n' + 'save_' + child_name + '\n')
                self.block_to_string(child_ref,child_name,outstring,4)    
                outstring.write('\n' + 'save_'+ '\n')   
            if self.standard != 'Dic':              #put contents after save frames
                self[blockref].SetOutputLength(wraplength,maxoutlength)
                outstring.write(str(self[blockref]))
        returnstring =  outstring.getvalue()
        outstring.close()
        return returnstring

    def block_to_string(self,block_ref,block_id,outstring,indentlevel=0):
        """Output a complete datablock indexed by [[block_ref]] and named [[block_id]], including children"""
        child_names = [(a[0],a[1].block_id) for a in self.child_table.items() if a[1].parent==block_ref]
        if self.standard == 'Dic':
            outstring.write(str(self[block_ref]))
        for child_ref,child_name in child_names:
            outstring.write('\n' + 'save_' + child_name + '\n')
            self.block_to_string(child_ref,child_name,outstring,indentlevel)
            outstring.write('\n' + '  '*indentlevel + 'save_' + '\n')
        if self.standard != 'Dic':
            outstring.write(str(self[block_ref]))

Ancestors (in MRO)

Instance variables

var PC

var blocktype

var characterset

var child_table

var dictionary

var header_comment

var lower_keys

var parent_id

var renamed

var scoping

var standard

var visible_keys

Methods

def __init__(

self, datasource=None, standard='CIF', blocktype=<class '__pdoc_file_module__.StarBlock'>, characterset='ascii', scoping='instance', parent_id=None, **kwargs)

def __init__(self,datasource=None,standard='CIF',blocktype = StarBlock,
             characterset='ascii',scoping='instance',parent_id=None,**kwargs):
    import collections
    self.dictionary = {}
    self.standard = standard
    self.lower_keys = set()           # short_cuts
    self.renamed = {}
    self.characterset = characterset
    self.PC = collections.namedtuple('PC',['block_id','parent'])
    self.child_table = {}
    self.visible_keys = []            # for efficiency
    self.parent_id = parent_id
    self.scoping = scoping  #will trigger setting of child table
    self.blocktype = blocktype
    if isinstance(datasource,BlockCollection):
        self.merge_fast(datasource)
        self.scoping = scoping   #reset visibility
    elif isinstance(datasource,dict):
        for key,value in datasource.items():
             self[key]= value
    self.header_comment = ''

def NewBlock(

self, blockname, blockcontents=None, fix=True, parent=None)

def NewBlock(self,blockname,blockcontents=None,fix=True,parent=None):
    if blockcontents is None:
        blockcontents = StarBlock()
    if self.standard == "CIF":
        blockcontents.setmaxnamelength(75)
    if len(blockname)>75:
             raise StarError , 'Blockname %s is longer than 75 characters' % blockname
    if fix:
        newblockname = re.sub('[  \t]','_',blockname)
    else: newblockname = blockname
    new_lowerbn = newblockname.lower()
    if new_lowerbn in self.lower_keys:
        if self.standard is not None:    #already there
           toplevelnames = [a[0] for a in self.child_table.items() if a[1].parent==None]
           if parent is None and new_lowerbn not in toplevelnames:  #can give a new key to this one
              while new_lowerbn in self.lower_keys: new_lowerbn = new_lowerbn + '+'
           elif parent is not None and new_lowerbn in toplevelnames: #can fix a different one
              replace_name = new_lowerbn            
              while replace_name in self.lower_keys: replace_name = replace_name + '+'
              self._rekey(new_lowerbn,replace_name)
              # now continue on to add in the new block
              if parent.lower() == new_lowerbn:        #the new block's requested parent just got renamed!!
                  parent = replace_name
           else:
              raise StarError( "Attempt to replace existing block " + blockname)
        else:
           del self[new_lowerbn]
    self.dictionary.update({new_lowerbn:blockcontents})
    self.lower_keys.add(new_lowerbn)
    if parent is None:
       self.child_table[new_lowerbn]=self.PC(newblockname,None)
       self.visible_keys.append(new_lowerbn)
    else:
       if parent.lower() in self.lower_keys:
          if self.scoping == 'instance':
             self.child_table[new_lowerbn]=self.PC(newblockname,parent.lower())
          else:
             self.child_table[new_lowerbn]=self.PC(newblockname,parent.lower())
             self.visible_keys.append(new_lowerbn)
       else:
           print 'Warning:Parent block %s does not exist for child %s' % (parent,newblockname)  
    return new_lowerbn  #in case calling routine wants to know

def WriteOut(

self, comment='', wraplength=80, maxoutlength=2048)

def WriteOut(self,comment='',wraplength=80,maxoutlength=2048):
    import cStringIO
    if not comment:
        comment = self.header_comment
    outstring = cStringIO.StringIO()
    outstring.write(comment)
    # loop over top-level
    top_block_names = [(a[0],a[1].block_id) for a in self.child_table.items() if a[1].parent is None]
    for blockref,blockname in top_block_names:
        outstring.write('\n' + 'data_' +blockname+'\n')
        child_names = [(a[0],a[1].block_id) for a in self.child_table.items() if a[1].parent==blockref]
        if self.standard == 'Dic':              #put contents before save frames
          self[blockref].SetOutputLength(wraplength,maxoutlength)
          outstring.write(str(self[blockref]))
        for child_ref,child_name in child_names:
            outstring.write('\n' + 'save_' + child_name + '\n')
            self.block_to_string(child_ref,child_name,outstring,4)    
            outstring.write('\n' + 'save_'+ '\n')   
        if self.standard != 'Dic':              #put contents after save frames
            self[blockref].SetOutputLength(wraplength,maxoutlength)
            outstring.write(str(self[blockref]))
    returnstring =  outstring.getvalue()
    outstring.close()
    return returnstring

def block_to_string(

self, block_ref, block_id, outstring, indentlevel=0)

Output a complete datablock indexed by [[block_ref]] and named [[block_id]], including children

def block_to_string(self,block_ref,block_id,outstring,indentlevel=0):
    """Output a complete datablock indexed by [[block_ref]] and named [[block_id]], including children"""
    child_names = [(a[0],a[1].block_id) for a in self.child_table.items() if a[1].parent==block_ref]
    if self.standard == 'Dic':
        outstring.write(str(self[block_ref]))
    for child_ref,child_name in child_names:
        outstring.write('\n' + 'save_' + child_name + '\n')
        self.block_to_string(child_ref,child_name,outstring,indentlevel)
        outstring.write('\n' + '  '*indentlevel + 'save_' + '\n')
    if self.standard != 'Dic':
        outstring.write(str(self[block_ref]))

def checknamelengths(

self, target_block, maxlength=-1)

def checknamelengths(self,target_block,maxlength=-1):
    if maxlength < 0:
        return
    else:
        toolong = filter(lambda a:len(a)>maxlength, target_block.keys())
    outstring = ""
    for it in toolong: outstring += "\n" + it
    if toolong:
       raise StarError( 'Following data names too long:' + outstring)

def clear(

self)

def clear(self):
    self.dictionary.clear()
    self.lower_keys = set()
    self.child_table = {}
    self.visible_keys = []

def copy(

self)

def copy(self):   
    newcopy = self.dictionary.copy()  #all blocks
    newcopy = BlockCollection('',newcopy,parent_id=self.parent_id)
    newcopy.child_table = self.child_table.copy()
    newcopy.lower_keys = self.lower_keys
    newcopy.characterset = self.characterset
    newcopy.scoping = self.scoping  #this sets visible keys
    return newcopy

def first_block(

self)

Return the 'first' block. This is not necessarily the first block in the file.

def first_block(self):
    """Return the 'first' block.  This is not necessarily the first block in the file."""
    if self.keys():
        return self[self.keys()[0]]

def get(

self, key, default=None)

def get(self,key,default=None):
    if self.has_key(key):     # take account of case
        return self.__getitem__(key)
    else:
        return default

def get_all(

self, item_name)

def get_all(self,item_name):
    raw_values = map(lambda a:self[a].get(item_name),self.keys())
    raw_values = filter(lambda a:a != None, raw_values)
    ret_vals = []
    for rv in raw_values:
        if isinstance(rv,list):
            for rvv in rv:
                if rvv not in ret_vals: ret_vals.append(rvv)
        else:
            if rv not in ret_vals: ret_vals.append(rv)
    return ret_vals

def get_child_list(

self, parentname)

Get a list of all child categories

def get_child_list(self,parentname):
    """Get a list of all child categories"""
    child_handles = [a[0] for a in self.child_table.items() if self.is_child_of_parent(parentname.lower(),a[0])]
    return child_handles

def get_children(

self, blockname, include_parent=False, scoping='dictionary')

Get all children of [[blockname]] as a block collection. If [[include_parent]] is True, the parent block will also be included in the block collection as the root.

def get_children(self,blockname,include_parent=False,scoping='dictionary'):
    """Get all children of [[blockname]] as a block collection. If [[include_parent]] is
    True, the parent block will also be included in the block collection as the root."""
    newbc = BlockCollection()
    block_lower = blockname.lower()
    proto_child_table = [a for a in self.child_table.items() if self.is_child_of_parent(block_lower,a[1].block_id)]
    newbc.child_table = dict(proto_child_table)
    if not include_parent:
       newbc.child_table.update(dict([(a[0],self.PC(a[1].block_id,None)) for a in proto_child_table if a[1].parent == block_lower]))
    newbc.lower_keys = set([a[0] for a in proto_child_table])
    newbc.dictionary = dict((a[0],self.dictionary[a[0]]) for a in proto_child_table)
    if include_parent:
        newbc.child_table.update({block_lower:self.PC(self.child_table[block_lower].block_id,None)})
        newbc.lower_keys.add(block_lower)
        newbc.dictionary.update({block_lower:self.dictionary[block_lower]})
    newbc.scoping = scoping
    return newbc

def get_immediate_children(

self, parentname)

Get the next level of children of the given block as a list, without nested levels

def get_immediate_children(self,parentname):
    """Get the next level of children of the given block as a list, without nested levels"""
    child_handles = [a for a in self.child_table.items() if a[1].parent == parentname.lower()]
    return child_handles

def get_parent(

self, blockname)

Return the name of the block enclosing [[blockname]] in canonical form (lower case)

def get_parent(self,blockname):
    """Return the name of the block enclosing [[blockname]] in canonical form (lower case)"""
    possibles = (a for a in self.child_table.items() if a[0] == blockname.lower())
    try:
        first = possibles.next()   #get first one
    except:
        raise StarError('no parent for %s' % blockname)
    try:
       second = possibles.next()
    except StopIteration:
       return first[1].parent
    raise StarError('More than one parent for %s' % blockname)

def get_roots(

self)

Get the top-level blocks

def get_roots(self):
    """Get the top-level blocks"""
    return [a for a in self.child_table.items() if a[1].parent==None]

def has_key(

self, key)

def has_key(self,key):
    if not isinstance(key,basestring): return 0
    if key.lower() in self.visible_keys:
        return 1
    return 0

def is_child_of_parent(

self, parentname, blockname)

Recursively search for children of blockname, case is important for now

def is_child_of_parent(self,parentname,blockname):
    """Recursively search for children of blockname, case is important for now"""
    checkname = parentname.lower()
    more_children = [a[0] for a in self.child_table.items() if a[1].parent == checkname]
    if blockname.lower() in more_children:
       return True
    else:
       for one_child in more_children:
           if self.is_child_of_parent(one_child,blockname): return True
    return False

def items(

self)

def items(self):
    return [(a,self[a]) for a in self.keys()]

def keys(

self)

def keys(self):
    return self.visible_keys

def lock(

self)

Disallow overwriting for all blocks in this collection

def lock(self):
    """Disallow overwriting for all blocks in this collection"""
    for a in self.lower_keys:
        self[a].overwrite = False

def merge(

self, new_bc, mode=None, parent=None, single_block=[], idblock='', match_att=[], match_function=None)

def merge(self,new_bc,mode=None,parent=None,single_block=[],
               idblock="",match_att=[],match_function=None):
    if mode is None:
        if self.standard is None:
           mode = 'replace'
        else:
           mode = 'strict'
    if single_block:
        self[single_block[0]].merge(new_bc[single_block[1]],mode,
                                               match_att=match_att,
                                               match_function=match_function)
        return None
    base_keys = [a[1].block_id for a in self.child_table.items()]
    block_to_item = base_keys   #default
    new_keys = [a[1].block_id for a in new_bc.child_table.items()]    #get list of incoming blocks
    if match_att:
        #make a blockname -> item name map
        if match_function:
            block_to_item = map(lambda a:match_function(self[a]),self.keys())
        else:
            block_to_item = map(lambda a:self[a].get(match_att[0],None),self.keys())
        #print `block_to_item`
    for key in new_keys:        #run over incoming blocknames
        if key == idblock: continue    #skip dictionary id
        basekey = key           #default value
        if len(match_att)>0:
           attval = new_bc[key].get(match_att[0],0)  #0 if ignoring matching
        else:
           attval = 0
        for ii in range(len(block_to_item)):  #do this way to get looped names
            thisatt = block_to_item[ii]       #keyname in old block
            #print "Looking for %s in %s" % (attval,thisatt)
            if attval == thisatt or \
               (isinstance(thisatt,list) and attval in thisatt):
                  basekey = base_keys.pop(ii)
                  block_to_item.remove(thisatt)
                  break
        if not self.has_key(basekey) or mode=="replace":
            new_parent = new_bc.get_parent(key)
            if parent is not None and new_parent is None:
               new_parent = parent
            self.NewBlock(basekey,new_bc[key],parent=new_parent)   #add the block
        else:
            if mode=="strict":
                raise StarError( "In strict merge mode: block %s in old and block %s in new files" % (basekey,key))
            elif mode=="overlay":
                # print "Merging block %s with %s" % (basekey,key)
                self[basekey].merge(new_bc[key],mode,match_att=match_att)
            else:  
                raise StarError( "Merge called with unknown mode %s" % mode)

def merge_fast(

self, new_bc, parent=None)

Do a fast merge

def merge_fast(self,new_bc,parent=None):
    """Do a fast merge"""
    if self.standard is None:
        mode = 'replace' 
    else:
        mode = 'strict'
    overlap_flag = not self.lower_keys.isdisjoint(new_bc.lower_keys)
    if overlap_flag and mode != 'replace':
        double_keys = self.lower_keys.intersection(new_bc.lower_keys)
        for dup_key in double_keys:
              our_parent = self.child_table[dup_key].parent
              their_parent = new_bc.child_table[dup_key].parent
              if (our_parent is None and their_parent is not None and parent is None) or\
                  parent is not None:  #rename our block
                start_key = dup_key
                while start_key in self.lower_keys: start_key = start_key+'+'
                self._rekey(dup_key,start_key)
                if parent.lower() == dup_key:  #we just renamed the prospective parent!
                    parent = start_key
              elif our_parent is not None and their_parent is None and parent is None:
                start_key = dup_key
                while start_key in new_bc.lower_keys: start_key = start_key+'+'
                new_bc._rekey(dup_key,start_key)
              else: 
                raise StarError("In strict merge mode:duplicate keys %s" % dup_key)
    self.dictionary.update(new_bc.dictionary) 
    self.lower_keys.update(new_bc.lower_keys)
    self.visible_keys += (list(new_bc.lower_keys))
    self.child_table.update(new_bc.child_table)
    if parent is not None:     #redo the child_table entries
          reparent_list = [(a[0],a[1].block_id) for a in new_bc.child_table.items() if a[1].parent==None]
          reparent_dict = [(a[0],self.PC(a[1],parent.lower())) for a in reparent_list]
          self.child_table.update(dict(reparent_dict))

def rename(

self, oldname, newname)

Rename datablock from [[oldname]] to [[newname]]. Both key and printed name are changed. No conformance checks are conducted.

def rename(self,oldname,newname):
    """Rename datablock from [[oldname]] to [[newname]]. Both key and printed name are changed.  No
       conformance checks are conducted."""
    realoldname = oldname.lower()
    realnewname = newname.lower()
    if realnewname in self.lower_keys:
        raise StarError,'Cannot change blockname %s to %s as %s already present' % (oldname,newname,newname)
    if realoldname not in self.lower_keys:
        raise KeyError,'Cannot find old block %s' % realoldname
    self._rekey(realoldname,realnewname,block_id=newname)

def set_parent(

self, parentname, childname)

Set the parent block

def set_parent(self,parentname,childname):
    """Set the parent block"""
    # first check that both blocks exist
    if parentname.lower() not in self.lower_keys:
        raise KeyError('Parent block %s does not exist' % parentname)
    if childname.lower() not in self.lower_keys:
        raise KeyError('Child block %s does not exist' % childname)
    old_entry = self.child_table[childname.lower()]
    self.child_table[childname.lower()]=self.PC(old_entry.block_id,
           parentname.lower())
    self.scoping = self.scoping #reset visibility

def unlock(

self)

Allow overwriting of all blocks in this collection

def unlock(self):
    """Allow overwriting of all blocks in this collection"""
    for a in self.lower_keys:
        self[a].overwrite=True

def update(

self, adict)

def update(self,adict):
    for key in adict.keys():
        self[key] = adict[key]

class CIFStringIO

class CIFStringIO(StringIO):
    def __init__(self,target_width=80,**kwargs):
        StringIO.__init__(self,**kwargs)
        self.currentpos = 0
        self.target_width = target_width
        self.tabwidth = -1
        self.indentlist = [0]

    def write(self,outstring,canbreak=False,mustbreak=False,do_tab=True,newindent=False,unindent=False,startcol=-1):
        """Write a string with correct linebreak, tabs and indents"""
        # do we need to break?
        if mustbreak:    #insert a new line and indent
            StringIO.write(self,'\n' + ' '*self.indentlist[-1])
            self.currentpos = self.indentlist[-1]
        if self.currentpos+len(outstring)>self.target_width: #try to break
            if canbreak:
                StringIO.write(self,'\n'+' '*self.indentlist[-1])
                self.currentpos = self.indentlist[-1]
        if newindent:           #indent by current amount
            if self.indentlist[-1] == 0:    #first time
                self.indentlist.append(self.currentpos)
                print 'Indentlist: ' + `self.indentlist`
            else:
                self.indentlist.append(self.indentlist[-1]+2)
        elif unindent:
            if len(self.indentlist)>1:
                self.indentlist.pop()
            else:
                print 'Warning: cannot unindent any further'
        #handle tabs
        if self.tabwidth >0 and do_tab:
            next_stop = ((self.currentpos//self.tabwidth)+1)*self.tabwidth
            #print 'Currentpos %d: Next tab stop at %d' % (self.currentpos,next_stop)
            if self.currentpos < next_stop:
                StringIO.write(self,(next_stop-self.currentpos)*' ')
                self.currentpos = next_stop
        #now output the string
        StringIO.write(self,outstring)
        last_line_break = outstring.rfind('\n')
        if last_line_break >=0:
            self.currentpos = len(outstring)-last_line_break
        else:
            self.currentpos = self.currentpos + len(outstring)
        
    def set_tab(self,tabwidth):
        """Set the tab stop position"""
        self.tabwidth = tabwidth

Ancestors (in MRO)

Instance variables

var currentpos

var indentlist

var tabwidth

var target_width

Methods

def __init__(

self, target_width=80, **kwargs)

def __init__(self,target_width=80,**kwargs):
    StringIO.__init__(self,**kwargs)
    self.currentpos = 0
    self.target_width = target_width
    self.tabwidth = -1
    self.indentlist = [0]

def close(

self)

Free the memory buffer.

def close(self):
    """Free the memory buffer.
    """
    if not self.closed:
        self.closed = True
        del self.buf, self.pos

def flush(

self)

Flush the internal buffer

def flush(self):
    """Flush the internal buffer
    """
    _complain_ifclosed(self.closed)

def getvalue(

self)

Retrieve the entire contents of the "file" at any time before the StringIO object's close() method is called.

The StringIO object can accept either Unicode or 8-bit strings, but mixing the two may take some care. If both are used, 8-bit strings that cannot be interpreted as 7-bit ASCII (that use the 8th bit) will cause a UnicodeError to be raised when getvalue() is called.

def getvalue(self):
    """
    Retrieve the entire contents of the "file" at any time before
    the StringIO object's close() method is called.
    The StringIO object can accept either Unicode or 8-bit strings,
    but mixing the two may take some care. If both are used, 8-bit
    strings that cannot be interpreted as 7-bit ASCII (that use the
    8th bit) will cause a UnicodeError to be raised when getvalue()
    is called.
    """
    _complain_ifclosed(self.closed)
    if self.buflist:
        self.buf += ''.join(self.buflist)
        self.buflist = []
    return self.buf

def isatty(

self)

Returns False because StringIO objects are not connected to a tty-like device.

def isatty(self):
    """Returns False because StringIO objects are not connected to a
    tty-like device.
    """
    _complain_ifclosed(self.closed)
    return False

def next(

self)

A file object is its own iterator, for example iter(f) returns f (unless f is closed). When a file is used as an iterator, typically in a for loop (for example, for line in f: print line), the next() method is called repeatedly. This method returns the next input line, or raises StopIteration when EOF is hit.

def next(self):
    """A file object is its own iterator, for example iter(f) returns f
    (unless f is closed). When a file is used as an iterator, typically
    in a for loop (for example, for line in f: print line), the next()
    method is called repeatedly. This method returns the next input line,
    or raises StopIteration when EOF is hit.
    """
    _complain_ifclosed(self.closed)
    r = self.readline()
    if not r:
        raise StopIteration
    return r

def read(

self, n=-1)

Read at most size bytes from the file (less if the read hits EOF before obtaining size bytes).

If the size argument is negative or omitted, read all data until EOF is reached. The bytes are returned as a string object. An empty string is returned when EOF is encountered immediately.

def read(self, n = -1):
    """Read at most size bytes from the file
    (less if the read hits EOF before obtaining size bytes).
    If the size argument is negative or omitted, read all data until EOF
    is reached. The bytes are returned as a string object. An empty
    string is returned when EOF is encountered immediately.
    """
    _complain_ifclosed(self.closed)
    if self.buflist:
        self.buf += ''.join(self.buflist)
        self.buflist = []
    if n is None or n < 0:
        newpos = self.len
    else:
        newpos = min(self.pos+n, self.len)
    r = self.buf[self.pos:newpos]
    self.pos = newpos
    return r

def readline(

self, length=None)

Read one entire line from the file.

A trailing newline character is kept in the string (but may be absent when a file ends with an incomplete line). If the size argument is present and non-negative, it is a maximum byte count (including the trailing newline) and an incomplete line may be returned.

An empty string is returned only when EOF is encountered immediately.

Note: Unlike stdio's fgets(), the returned string contains null characters ('\0') if they occurred in the input.

def readline(self, length=None):
    r"""Read one entire line from the file.
    A trailing newline character is kept in the string (but may be absent
    when a file ends with an incomplete line). If the size argument is
    present and non-negative, it is a maximum byte count (including the
    trailing newline) and an incomplete line may be returned.
    An empty string is returned only when EOF is encountered immediately.
    Note: Unlike stdio's fgets(), the returned string contains null
    characters ('\0') if they occurred in the input.
    """
    _complain_ifclosed(self.closed)
    if self.buflist:
        self.buf += ''.join(self.buflist)
        self.buflist = []
    i = self.buf.find('\n', self.pos)
    if i < 0:
        newpos = self.len
    else:
        newpos = i+1
    if length is not None and length >= 0:
        if self.pos + length < newpos:
            newpos = self.pos + length
    r = self.buf[self.pos:newpos]
    self.pos = newpos
    return r

def readlines(

self, sizehint=0)

Read until EOF using readline() and return a list containing the lines thus read.

If the optional sizehint argument is present, instead of reading up to EOF, whole lines totalling approximately sizehint bytes (or more to accommodate a final whole line).

def readlines(self, sizehint = 0):
    """Read until EOF using readline() and return a list containing the
    lines thus read.
    If the optional sizehint argument is present, instead of reading up
    to EOF, whole lines totalling approximately sizehint bytes (or more
    to accommodate a final whole line).
    """
    total = 0
    lines = []
    line = self.readline()
    while line:
        lines.append(line)
        total += len(line)
        if 0 < sizehint <= total:
            break
        line = self.readline()
    return lines

def seek(

self, pos, mode=0)

Set the file's current position.

The mode argument is optional and defaults to 0 (absolute file positioning); other values are 1 (seek relative to the current position) and 2 (seek relative to the file's end).

There is no return value.

def seek(self, pos, mode = 0):
    """Set the file's current position.
    The mode argument is optional and defaults to 0 (absolute file
    positioning); other values are 1 (seek relative to the current
    position) and 2 (seek relative to the file's end).
    There is no return value.
    """
    _complain_ifclosed(self.closed)
    if self.buflist:
        self.buf += ''.join(self.buflist)
        self.buflist = []
    if mode == 1:
        pos += self.pos
    elif mode == 2:
        pos += self.len
    self.pos = max(0, pos)

def set_tab(

self, tabwidth)

Set the tab stop position

def set_tab(self,tabwidth):
    """Set the tab stop position"""
    self.tabwidth = tabwidth

def tell(

self)

Return the file's current position.

def tell(self):
    """Return the file's current position."""
    _complain_ifclosed(self.closed)
    return self.pos

def truncate(

self, size=None)

Truncate the file's size.

If the optional size argument is present, the file is truncated to (at most) that size. The size defaults to the current position. The current file position is not changed unless the position is beyond the new file size.

If the specified size exceeds the file's current size, the file remains unchanged.

def truncate(self, size=None):
    """Truncate the file's size.
    If the optional size argument is present, the file is truncated to
    (at most) that size. The size defaults to the current position.
    The current file position is not changed unless the position
    is beyond the new file size.
    If the specified size exceeds the file's current size, the
    file remains unchanged.
    """
    _complain_ifclosed(self.closed)
    if size is None:
        size = self.pos
    elif size < 0:
        raise IOError(EINVAL, "Negative size not allowed")
    elif size < self.pos:
        self.pos = size
    self.buf = self.getvalue()[:size]
    self.len = size

def write(

self, outstring, canbreak=False, mustbreak=False, do_tab=True, newindent=False, unindent=False, startcol=-1)

Write a string with correct linebreak, tabs and indents

def write(self,outstring,canbreak=False,mustbreak=False,do_tab=True,newindent=False,unindent=False,startcol=-1):
    """Write a string with correct linebreak, tabs and indents"""
    # do we need to break?
    if mustbreak:    #insert a new line and indent
        StringIO.write(self,'\n' + ' '*self.indentlist[-1])
        self.currentpos = self.indentlist[-1]
    if self.currentpos+len(outstring)>self.target_width: #try to break
        if canbreak:
            StringIO.write(self,'\n'+' '*self.indentlist[-1])
            self.currentpos = self.indentlist[-1]
    if newindent:           #indent by current amount
        if self.indentlist[-1] == 0:    #first time
            self.indentlist.append(self.currentpos)
            print 'Indentlist: ' + `self.indentlist`
        else:
            self.indentlist.append(self.indentlist[-1]+2)
    elif unindent:
        if len(self.indentlist)>1:
            self.indentlist.pop()
        else:
            print 'Warning: cannot unindent any further'
    #handle tabs
    if self.tabwidth >0 and do_tab:
        next_stop = ((self.currentpos//self.tabwidth)+1)*self.tabwidth
        #print 'Currentpos %d: Next tab stop at %d' % (self.currentpos,next_stop)
        if self.currentpos < next_stop:
            StringIO.write(self,(next_stop-self.currentpos)*' ')
            self.currentpos = next_stop
    #now output the string
    StringIO.write(self,outstring)
    last_line_break = outstring.rfind('\n')
    if last_line_break >=0:
        self.currentpos = len(outstring)-last_line_break
    else:
        self.currentpos = self.currentpos + len(outstring)

def writelines(

self, iterable)

Write a sequence of strings to the file. The sequence can be any iterable object producing strings, typically a list of strings. There is no return value.

(The name is intended to match readlines(); writelines() does not add line separators.)

def writelines(self, iterable):
    """Write a sequence of strings to the file. The sequence can be any
    iterable object producing strings, typically a list of strings. There
    is no return value.
    (The name is intended to match readlines(); writelines() does not add
    line separators.)
    """
    write = self.write
    for line in iterable:
        write(line)

class LoopBlock

class LoopBlock(object):
    def __init__(self,parent_block,dataname):
        self.loop_no = parent_block.FindLoop(dataname)
        if self.loop_no < 0:
            raise KeyError, '%s is not in a loop structure' % dataname
        self.parent_block = parent_block

    def keys(self):
        return self.parent_block.loops[self.loop_no]

    def values(self):
        return [self.parent_block[a] for a in self.keys()]

    def items(self):
        return zip(self.keys(),self.values())

    def __getitem__(self,dataname):
        if isinstance(dataname,int):   #a packet request
            return self.GetPacket(dataname)
        if dataname in self.keys():
            return self.parent_block[dataname]
        else:
            raise KeyError, '%s not in loop block' % dataname

    def __setitem__(self,dataname,value):
        self.parent_block[dataname] = value
        self.parent_block.AddLoopName(self.keys()[0],dataname)

    def has_key(self,key):
        return key in self.parent_block.loops[self.loop_no]

    def __iter__(self):
        packet_list = zip(*self.values())
        names = self.keys()
        for p in packet_list:
            r = StarPacket(p)
            for n in range(len(names)):
                setattr(r,names[n].lower(),r[n])
            yield r

    # for compatibility
    def __getattr__(self,attname):
        return getattr(self.parent_block,attname)

    def load_iter(self,coords=[]):
        count = 0        #to create packet index 
        while not self.popout:
            # ok, we have a new packet:  append a list to our subloops
            for aloop in self.loops:
                aloop.new_enclosing_packet()
            for iname in self.item_order:
                if isinstance(iname,LoopBlock):       #into a nested loop
                    for subitems in iname.load_iter(coords=coords+[count]):
                        # print 'Yielding %s' % `subitems`
                        yield subitems
                    # print 'End of internal loop'
                else:
                    if self.dimension == 0:
                        # print 'Yielding %s' % `self[iname]`
                        yield self,self[iname]
                    else:
                        backval = self.block[iname]
                        for i in range(len(coords)):
                           # print 'backval, coords: %s, %s' % (`backval`,`coords`)
                           backval = backval[coords[i]]
                        yield self,backval
            count = count + 1      # count packets
        self.popout = False        # reinitialise
        # print 'Finished iterating'
        yield self,'###Blank###'     #this value should never be used

    # an experimental fast iterator for level-1 loops (ie CIF)
    def fast_load_iter(self):
        targets = map(lambda a:self.block[a],self.item_order)
        while targets:
            for target in targets:
                yield self,target

    # Add another list of the required shape to take into account a new outer packet
    def new_enclosing_packet(self):
        if self.dimension > 1:      #otherwise have a top-level list
            for iname in self.keys():  #includes lower levels
                target_list = self[iname]
                for i in range(3,self.dimension): #dim 2 upwards are lists of lists of... 
                    target_list = target_list[-1]
                target_list.append([])
                # print '%s now %s' % (iname,`self[iname]`)

    def recursive_iter(self,dict_so_far={},coord=[]):
        # print "Recursive iter: coord %s, keys %s, dim %d" % (`coord`,`self.block.keys()`,self.dimension)
        my_length = 0
        top_items = self.block.items()
        top_values = self.block.values()       #same order as items
        drill_values = self.block.values()
        for dimup in range(0,self.dimension):  #look higher in the tree
            if len(drill_values)>0:            #this block has values
                drill_values=drill_values[0]   #drill in
            else:
                raise StarError("Malformed loop packet %s" % `top_items[0]`)
        my_length = len(drill_values[0])       #length of 'string' entry
        if self.dimension == 0:                #top level
            for aloop in self.loops:
                for apacket in aloop.recursive_iter():
                    # print "Recursive yielding %s" % `dict(top_items + apacket.items())`
                    prep_yield = StarPacket(top_values+apacket.values())  #straight list
                    for name,value in top_items + apacket.items():
                        setattr(prep_yield,name,value)
                    yield prep_yield
        else:                                  #in some loop
            for i in range(my_length):
                kvpairs = map(lambda a:(a,self.coord_to_group(a,coord)[i]),self.block.keys())
                kvvals = map(lambda a:a[1],kvpairs)   #just values
                # print "Recursive kvpairs at %d: %s" % (i,`kvpairs`)
                if self.loops:
                  for aloop in self.loops:
                    for apacket in aloop.recursive_iter(coord=coord+[i]):
                        # print "Recursive yielding %s" % `dict(kvpairs + apacket.items())`
                        prep_yield = StarPacket(kvvals+apacket.values())
                        for name,value in kvpairs + apacket.items():
                            setattr(prep_yield,name,value)
                        yield prep_yield
                else:           # we're at the bottom of the tree
                    # print "Recursive yielding %s" % `dict(kvpairs)`
                    prep_yield = StarPacket(kvvals)
                    for name,value in kvpairs:
                        setattr(prep_yield,name,value)
                    yield prep_yield

    # small function to use the coordinates. 
    def coord_to_group(self,dataname,coords):
          if not isinstance(dataname,basestring):
             return dataname     # flag inner loop processing
          newm = self[dataname]          # newm must be a list or tuple
          for c in coords:
              # print "Coord_to_group: %s ->" % (`newm`),
              newm = newm[c]
              # print `newm`
          return newm 

    def flat_iterator(self):
        if self.dimension == 0:   
            yield copy.copy(self)
        else:
            my_length = 0
            top_keys = self.block.keys()
            if len(top_keys)>0:
                my_length = len(self.block[top_keys[0]])
            for pack_no in range(my_length):
                yield(self.collapse(pack_no))
            

    def RemoveItem(self,itemname):
        # first check any loops
        loop_no = self.FindLoop(itemname)
        testkey = itemname.lower()
        if self.has_key(testkey):
            del self.block[testkey]
            del self.true_case[testkey]
            # now remove from loop
            if loop_no >= 0:
                self.loops[loop_no].remove(testkey)
                if len(self.loops[loop_no])==0:
                    del self.loops[loop_no]
                    self.item_order.remove(loop_no)
            else:  #will appear in order list
                self.item_order.remove(testkey)
     
    def RemoveLoopItem(self,itemname):
        self.RemoveItem(itemname)

    def GetLoop(self,keyname):
        """Return a LoopBlock object containing keyname"""
        return LoopBlock(self,keyname)

    def GetPacket(self,index):
        thispack = StarPacket([])
        for myitem in self.parent_block.loops[self.loop_no]:
            thispack.append(self[myitem][index])
            setattr(thispack,myitem,thispack[-1])
        return thispack 

    def AddPacket(self,packet):
        for myitem in self.parent_block.loops[self.loop_no]:
            old_values = self.parent_block[myitem]
            old_values.append(packet.__getattribute__(myitem))
            self.parent_block[myitem] = old_values
        
    def GetItemOrder(self):
        return self.parent_block.loops[self.loop_no][:]


    def GetItemOrder(self):
        return self.parent_block.loops[self.loop_no][:]

    def ChangeItemOrder(self,itemname,newpos):
        self.parent_block.loops[self.loop_no].remove(itemname.lower())
        self.parent_block.loops[self.loop_no].insert(newpos,itemname.lower())

    def GetItemPosition(self,itemname):
        import string
        if isinstance(itemname,int):
            # return loop position
            return (-1, self.item_order.index(itemname))
        if not self.has_key(itemname):
            raise ValueError, 'No such dataname %s' % itemname
        testname = itemname.lower()
        if testname in self.item_order:
            return (-1,self.item_order.index(testname))
        loop_no = self.FindLoop(testname)
        loop_pos = self.loops[loop_no].index(testname)
        return loop_no,loop_pos

    def GetLoopNames(self,keyname):
        if keyname in self:
            return self.keys()
        for aloop in self.loops:
            try: 
                return aloop.GetLoopNames(keyname)
            except KeyError:
                pass
        raise KeyError, 'Item does not exist'

    def GetLoopNames(self,keyname):
        """Return all datanames appearing together with [[keyname]]"""
        loop_no = self.FindLoop(keyname)
        if loop_no >= 0:
            return self.loops[loop_no]
        else:
            raise KeyError, '%s is not in any loop' % keyname

    def AddToLoop(self,dataname,loopdata):
        thisloop = self.GetLoop(dataname)
        for itemname,itemvalue in loopdata.items():
            thisloop[itemname] = itemvalue 
        
    def AddToLoop(self,dataname,loopdata):
        # check lengths
        thisloop = self.FindLoop(dataname)
        loop_len = len(self[dataname])
        bad_vals = [a for a in loopdata.items() if len(a[1])!=loop_len]
        if len(bad_vals)>0:
           raise StarLengthError, "Number of values for looped datanames %s not equal to %d" \
               % (`bad_vals`,loop_len)
        self.update(loopdata)
        self.loops[thisloop]+=loopdata.keys()

Ancestors (in MRO)

Instance variables

var loop_no

var parent_block

Methods

def __init__(

self, parent_block, dataname)

def __init__(self,parent_block,dataname):
    self.loop_no = parent_block.FindLoop(dataname)
    if self.loop_no < 0:
        raise KeyError, '%s is not in a loop structure' % dataname
    self.parent_block = parent_block

def AddPacket(

self, packet)

def AddPacket(self,packet):
    for myitem in self.parent_block.loops[self.loop_no]:
        old_values = self.parent_block[myitem]
        old_values.append(packet.__getattribute__(myitem))
        self.parent_block[myitem] = old_values

def AddToLoop(

self, dataname, loopdata)

def AddToLoop(self,dataname,loopdata):
    # check lengths
    thisloop = self.FindLoop(dataname)
    loop_len = len(self[dataname])
    bad_vals = [a for a in loopdata.items() if len(a[1])!=loop_len]
    if len(bad_vals)>0:
       raise StarLengthError, "Number of values for looped datanames %s not equal to %d" \
           % (`bad_vals`,loop_len)
    self.update(loopdata)
    self.loops[thisloop]+=loopdata.keys()

def ChangeItemOrder(

self, itemname, newpos)

def ChangeItemOrder(self,itemname,newpos):
    self.parent_block.loops[self.loop_no].remove(itemname.lower())
    self.parent_block.loops[self.loop_no].insert(newpos,itemname.lower())

def GetItemOrder(

self)

def GetItemOrder(self):
    return self.parent_block.loops[self.loop_no][:]

def GetItemPosition(

self, itemname)

def GetItemPosition(self,itemname):
    import string
    if isinstance(itemname,int):
        # return loop position
        return (-1, self.item_order.index(itemname))
    if not self.has_key(itemname):
        raise ValueError, 'No such dataname %s' % itemname
    testname = itemname.lower()
    if testname in self.item_order:
        return (-1,self.item_order.index(testname))
    loop_no = self.FindLoop(testname)
    loop_pos = self.loops[loop_no].index(testname)
    return loop_no,loop_pos

def GetLoop(

self, keyname)

Return a LoopBlock object containing keyname

def GetLoop(self,keyname):
    """Return a LoopBlock object containing keyname"""
    return LoopBlock(self,keyname)

def GetLoopNames(

self, keyname)

Return all datanames appearing together with [[keyname]]

def GetLoopNames(self,keyname):
    """Return all datanames appearing together with [[keyname]]"""
    loop_no = self.FindLoop(keyname)
    if loop_no >= 0:
        return self.loops[loop_no]
    else:
        raise KeyError, '%s is not in any loop' % keyname

def GetPacket(

self, index)

def GetPacket(self,index):
    thispack = StarPacket([])
    for myitem in self.parent_block.loops[self.loop_no]:
        thispack.append(self[myitem][index])
        setattr(thispack,myitem,thispack[-1])
    return thispack 

def RemoveItem(

self, itemname)

def RemoveItem(self,itemname):
    # first check any loops
    loop_no = self.FindLoop(itemname)
    testkey = itemname.lower()
    if self.has_key(testkey):
        del self.block[testkey]
        del self.true_case[testkey]
        # now remove from loop
        if loop_no >= 0:
            self.loops[loop_no].remove(testkey)
            if len(self.loops[loop_no])==0:
                del self.loops[loop_no]
                self.item_order.remove(loop_no)
        else:  #will appear in order list
            self.item_order.remove(testkey)

def RemoveLoopItem(

self, itemname)

def RemoveLoopItem(self,itemname):
    self.RemoveItem(itemname)

def coord_to_group(

self, dataname, coords)

def coord_to_group(self,dataname,coords):
      if not isinstance(dataname,basestring):
         return dataname     # flag inner loop processing
      newm = self[dataname]          # newm must be a list or tuple
      for c in coords:
          # print "Coord_to_group: %s ->" % (`newm`),
          newm = newm[c]
          # print `newm`
      return newm 

def fast_load_iter(

self)

def fast_load_iter(self):
    targets = map(lambda a:self.block[a],self.item_order)
    while targets:
        for target in targets:
            yield self,target

def flat_iterator(

self)

def flat_iterator(self):
    if self.dimension == 0:   
        yield copy.copy(self)
    else:
        my_length = 0
        top_keys = self.block.keys()
        if len(top_keys)>0:
            my_length = len(self.block[top_keys[0]])
        for pack_no in range(my_length):
            yield(self.collapse(pack_no))

def has_key(

self, key)

def has_key(self,key):
    return key in self.parent_block.loops[self.loop_no]

def items(

self)

def items(self):
    return zip(self.keys(),self.values())

def keys(

self)

def keys(self):
    return self.parent_block.loops[self.loop_no]

def load_iter(

self, coords=[])

def load_iter(self,coords=[]):
    count = 0        #to create packet index 
    while not self.popout:
        # ok, we have a new packet:  append a list to our subloops
        for aloop in self.loops:
            aloop.new_enclosing_packet()
        for iname in self.item_order:
            if isinstance(iname,LoopBlock):       #into a nested loop
                for subitems in iname.load_iter(coords=coords+[count]):
                    # print 'Yielding %s' % `subitems`
                    yield subitems
                # print 'End of internal loop'
            else:
                if self.dimension == 0:
                    # print 'Yielding %s' % `self[iname]`
                    yield self,self[iname]
                else:
                    backval = self.block[iname]
                    for i in range(len(coords)):
                       # print 'backval, coords: %s, %s' % (`backval`,`coords`)
                       backval = backval[coords[i]]
                    yield self,backval
        count = count + 1      # count packets
    self.popout = False        # reinitialise
    # print 'Finished iterating'
    yield self,'###Blank###'     #this value should never be used

def new_enclosing_packet(

self)

def new_enclosing_packet(self):
    if self.dimension > 1:      #otherwise have a top-level list
        for iname in self.keys():  #includes lower levels
            target_list = self[iname]
            for i in range(3,self.dimension): #dim 2 upwards are lists of lists of... 
                target_list = target_list[-1]
            target_list.append([])

def recursive_iter(

self, dict_so_far={}, coord=[])

def recursive_iter(self,dict_so_far={},coord=[]):
    # print "Recursive iter: coord %s, keys %s, dim %d" % (`coord`,`self.block.keys()`,self.dimension)
    my_length = 0
    top_items = self.block.items()
    top_values = self.block.values()       #same order as items
    drill_values = self.block.values()
    for dimup in range(0,self.dimension):  #look higher in the tree
        if len(drill_values)>0:            #this block has values
            drill_values=drill_values[0]   #drill in
        else:
            raise StarError("Malformed loop packet %s" % `top_items[0]`)
    my_length = len(drill_values[0])       #length of 'string' entry
    if self.dimension == 0:                #top level
        for aloop in self.loops:
            for apacket in aloop.recursive_iter():
                # print "Recursive yielding %s" % `dict(top_items + apacket.items())`
                prep_yield = StarPacket(top_values+apacket.values())  #straight list
                for name,value in top_items + apacket.items():
                    setattr(prep_yield,name,value)
                yield prep_yield
    else:                                  #in some loop
        for i in range(my_length):
            kvpairs = map(lambda a:(a,self.coord_to_group(a,coord)[i]),self.block.keys())
            kvvals = map(lambda a:a[1],kvpairs)   #just values
            # print "Recursive kvpairs at %d: %s" % (i,`kvpairs`)
            if self.loops:
              for aloop in self.loops:
                for apacket in aloop.recursive_iter(coord=coord+[i]):
                    # print "Recursive yielding %s" % `dict(kvpairs + apacket.items())`
                    prep_yield = StarPacket(kvvals+apacket.values())
                    for name,value in kvpairs + apacket.items():
                        setattr(prep_yield,name,value)
                    yield prep_yield
            else:           # we're at the bottom of the tree
                # print "Recursive yielding %s" % `dict(kvpairs)`
                prep_yield = StarPacket(kvvals)
                for name,value in kvpairs:
                    setattr(prep_yield,name,value)
                yield prep_yield

def values(

self)

def values(self):
    return [self.parent_block[a] for a in self.keys()]

class StarBlock

class StarBlock(object):
    def __init__(self,data = (), maxoutlength=2048, wraplength=80, overwrite=True,
                 characterset='ascii',maxnamelength=-1):
        self.block = {}    #the actual data storage (lower case keys)
        self.loops = {}    #each loop is indexed by a number and contains a list of datanames
        self.item_order = []  #lower case, loops referenced by integer
        self.formatting_hints = {}
        self.true_case = {} #transform lower case to supplied case
        self.provide_value = False   #prefer string version always
        self.dictionary = None      #DDLm dictionary
        self.popout = False         #used during load iteration
        self.curitem = -1           #used during iteration
        self.maxoutlength = maxoutlength
        self.setmaxnamelength(maxnamelength)  #to enforce CIF limit of 75 characters
        self.wraplength = wraplength
        self.overwrite = overwrite
        self.characterset = characterset
        if self.characterset == 'ascii':
            self.char_check = re.compile("[][ \n\r\t!%&\(\)*+,./:<=>?@0-9A-Za-z\\\\^`{}\|~\"#$';_-]+",re.M)
        elif self.characterset == 'unicode':
            self.char_check = re.compile(u"[][ \n\r\t!%&\(\)*+,./:<=>?@0-9A-Za-z\\\\^`{}\|~\"#$';_\u00A0-\uD7FF\uE000-\uFDCF\uFDF0-\uFFFD\U00010000-\U0010FFFD-]+",re.M)
        else:
            raise StarError("No character set specified")
        if isinstance(data,(tuple,list)):
            for item in data:
                self.AddLoopItem(item)
        elif isinstance(data,StarBlock):
            self.block = data.block.copy() 
            self.item_order = data.item_order[:]
            self.true_case = data.true_case.copy()
            # loops as well
            self.loops = data.loops.copy()

    def setmaxnamelength(self,maxlength):
        """Set the maximum allowable dataname length (-1 for no check)"""
        self.maxnamelength = maxlength
        if maxlength > 0:
            bad_names = [a for a in self.keys() if len(a)>self.maxnamelength]
            if len(bad_names)>0:
                raise StarError, 'Datanames too long: ' + `bad_names`

    def __str__(self):
        return self.printsection()

    def __setitem__(self,key,value):
        if key == "saves":
            raise StarError("""Setting the saves key is deprecated. Add the save block to
    an enclosing block collection (e.g. CIF or STAR file) with this block as child""")
        self.AddItem(key,value)

    def __getitem__(self,key):
        if key == "saves":
            raise StarError("""The saves key is deprecated. Access the save block from
    the enclosing block collection (e.g. CIF or STAR file object)""") 
        try:
           rawitem,is_value = self.GetFullItemValue(key)
        except KeyError:
           if self.dictionary:
               # send the dictionary the required key and a pointer to us
               new_value = self.dictionary.derive_item(key,self,store_value=True)
               print 'Set %s to derived value %s' % (key, `new_value`)
               return new_value
           else:
               raise KeyError, 'No such item: %s' % key
        # we now have an item, we can try to convert it to a number if that is appropriate
        # note numpy values are never stored but are converted to lists
        if not self.dictionary or not self.dictionary.has_key(key): return rawitem
        print '%s: is_value %s provide_value %s value %s' % (key,`is_value`,`self.provide_value`,`rawitem`)
        if is_value:
            if self.provide_value: return rawitem
            else:
               print 'Turning %s into string' % `rawitem`
               return self.convert_to_string(key)
        else:    # a string
            if self.provide_value and rawitem != '?' and rawitem != ".":
                return self.dictionary.change_type(key,rawitem)
            return rawitem   #can't do anything
           
    def __delitem__(self,key):
        self.RemoveItem(key)

    def __len__(self):
        blen = len(self.block)
        return blen    

    def __nonzero__(self):
        if self.__len__() > 0: return 1
        return 0

    # keys returns all internal keys
    def keys(self):
        return self.block.keys()    #always lower case

    def values(self):
        return [self[a] for a in self.keys()]

    def items(self):
        return [a for a in zip(self.keys(),self.values())]

    def has_key(self,key):
        if isinstance(key,basestring) and key.lower() in self.keys():
            return 1
        return 0

    def get(self,key,default=None):
        if self.has_key(key):
            retval = self.__getitem__(key)
        else:
            retval = default
        return retval

    def clear(self):
        self.block = {}
        self.loops = {}
        self.item_order = []
        self.true_case = {}

    # doesn't appear to work
    def copy(self):
        newcopy = StarBlock()
        newcopy.block = self.block.copy()
        newcopy.loops = []
        newcopy.item_order = self.item_order[:]
        newcopy.true_case = self.true_case.copy()
        newcopy.loops = self.loops.copy()
    #    return self.copy.im_class(newcopy)   #catch inheritance
        return newcopy
     
    def update(self,adict):
        for key in adict.keys():
            self.AddItem(key,adict[key])

    def GetItemPosition(self,itemname):
        import string
        if isinstance(itemname,int):
            # return loop position
            return (-1, self.item_order.index(itemname))
        if not self.has_key(itemname):
            raise ValueError, 'No such dataname %s' % itemname
        testname = itemname.lower()
        if testname in self.item_order:
            return (-1,self.item_order.index(testname))
        loop_no = self.FindLoop(testname)
        loop_pos = self.loops[loop_no].index(testname)
        return loop_no,loop_pos

    def ChangeItemOrder(self,itemname,newpos):
        """Move itemname to newpos"""
        if isinstance(itemname,basestring):
            true_name = itemname.lower()
        else:
            true_name = itemname
        self.item_order.remove(true_name)
        self.item_order.insert(newpos,true_name)

    def GetItemOrder(self):
        return self.item_order[:]

    def AddItem(self,key,value,precheck=False):
        if not isinstance(key,basestring):
             raise TypeError, 'Star datanames are strings only (got %s)' % `key`
        if not precheck:
             self.check_data_name(key,self.maxnamelength)    # make sure no nasty characters   
        # check for overwriting
        if self.has_key(key):
             if not self.overwrite:
                 raise StarError( 'Attempt to insert duplicate item name %s' % key)
        # put the data in the right container
        regval,empty_val = self.regularise_data(value)
        # check for pure string data
        pure_string = check_stringiness(regval)
        if not precheck:
            self.check_item_value(regval)
        # update ancillary information first
        lower_key = key.lower()
        if not self.has_key(lower_key) and self.FindLoop(lower_key)<0:      #need to add to order
            self.item_order.append(lower_key)
        # always remove from our case table in case the case is different
        else:
            del self.true_case[lower_key]
        self.true_case[lower_key] = key
        if pure_string:
            self.block.update({lower_key:[regval,empty_val]})
        else:
            self.block.update({lower_key:[empty_val,regval]})  
            
    def AddLoopItem(self,incomingdata,precheck=False,maxlength=-1):
        # print "Received data %s" % `incomingdata`
        # we accept tuples, strings, lists and dicts!!
        # Direct insertion: we have a string-valued key, with an array
        # of values -> single-item into our loop
        if isinstance(incomingdata[0],(tuple,list)):
           # a whole loop
           keyvallist = zip(incomingdata[0],incomingdata[1])
           for key,value in keyvallist:
               self.AddItem(key,value)
           self.CreateLoop(incomingdata[0])
        elif not isinstance(incomingdata[0],basestring):
             raise TypeError, 'Star datanames are strings only (got %s)' % `incomingdata[0]`
        else:
            self.AddItem(incomingdata[0],incomingdata[1])

    def check_data_name(self,dataname,maxlength=-1): 
        if maxlength > 0:
            self.check_name_length(dataname,maxlength)
        if dataname[0]!='_':
            raise StarError( 'Dataname ' + dataname + ' does not begin with _')
        if self.characterset=='ascii':
            if len (filter (lambda a: ord(a) < 33 or ord(a) > 126, dataname)) > 0:
                raise StarError( 'Dataname ' + dataname + ' contains forbidden characters')
        else:
            # print 'Checking %s for unicode characterset conformance' % dataname
            if len (filter (lambda a: ord(a) < 33, dataname)) > 0:
                raise StarError( 'Dataname ' + dataname + ' contains forbidden characters (below code point 33)')
            if len (filter (lambda a: ord(a) > 126 and ord(a) < 160, dataname)) > 0:
                raise StarError( 'Dataname ' + dataname + ' contains forbidden characters (between code point 127-159)')
            if len (filter (lambda a: ord(a) > 0xD7FF and ord(a) < 0xE000, dataname)) > 0:
                raise StarError( 'Dataname ' + dataname + ' contains unsupported characters (between U+D800 and U+E000)')
            if len (filter (lambda a: ord(a) > 0xFDCF and ord(a) < 0xFDF0, dataname)) > 0:
                raise StarError( 'Dataname ' + dataname + ' contains unsupported characters (between U+FDD0 and U+FDEF)')
            if len (filter (lambda a: ord(a) == 0xFFFE or ord(a) == 0xFFFF, dataname)) > 0:
                raise StarError( 'Dataname ' + dataname + ' contains unsupported characters (U+FFFE and/or U+FFFF)')
            if len (filter (lambda a: ord(a) > 0x10000 and (ord(a) & 0xE == 0xE) , dataname)) > 0:
                print '%s fails' % dataname
                for a in dataname: print '%x' % ord(a),
                print
                raise StarError( u'Dataname ' + dataname + u' contains unsupported characters (U+xFFFE and/or U+xFFFF)')

    def check_name_length(self,dataname,maxlength):
        if len(dataname)>maxlength:
            raise StarError( 'Dataname %s exceeds maximum length %d' % (dataname,maxlength))
        return

    def check_item_value(self,item):
        test_item = item
        if not isinstance(item,(list,dict,tuple)):
           test_item = [item]         #single item list
        def check_one (it):
            if isinstance(it,basestring):
                if it=='': return
                me = self.char_check.match(it)            
                if not me:
                    print "Fail value check: %s" % it
                    raise StarError, u'Bad character in %s' % it
                else:
                    if me.span() != (0,len(it)):
                        print "Fail value check, match only %d-%d in string %s" % (me.span()[0],me.span()[1],`it`)
                        raise StarError,u'Data item "' + `it` +  u'"... contains forbidden characters'
        map(check_one,test_item)

    def regularise_data(self,dataitem):
        """Place dataitem into a list if necessary"""
        from numbers import Number
        if isinstance(dataitem,(Number,basestring,StarList,StarDict)):
            return dataitem,None
        if isinstance(dataitem,(tuple,list)):
            return dataitem,[None]*len(dataitem)
        # so try to make into a list
        try:
            regval = list(dataitem)
        except TypeError, value:
            raise StarError( str(dataitem) + ' is wrong type for data value\n' )
        return regval,[None]*len(regval)
        
    def RemoveItem(self,itemname):
        # first check any loops
        loop_no = self.FindLoop(itemname)
        testkey = itemname.lower()
        if self.has_key(testkey):
            del self.block[testkey]
            del self.true_case[testkey]
            # now remove from loop
            if loop_no >= 0:
                self.loops[loop_no].remove(testkey)
                if len(self.loops[loop_no])==0:
                    del self.loops[loop_no]
                    self.item_order.remove(loop_no)
            else:  #will appear in order list
                self.item_order.remove(testkey)
     
    def RemoveLoopItem(self,itemname):
        self.RemoveItem(itemname)

    def GetItemValue(self,itemname):
        """Return value of itemname"""
        return self.GetFullItemValue(itemname)[0]

    def GetFullItemValue(self,itemname):
        """Return value of itemname and whether or not it is a native value"""
        try:
            s,v = self.block[itemname.lower()]
        except KeyError:
            raise KeyError, 'Itemname %s not in datablock' % itemname
        # prefer string value unless all are None
        # are we a looped value?
        if not isinstance(s,(tuple,list)) or isinstance(s,StarList):
            if s is not None or (isinstance(s,StarList) and None not in s): 
                return s,False    #a string value
            else:
                return v,not isinstance(v,StarList)  #a StarList is not calculation-ready
        elif None not in s: 
            return s,False     #a list of string values
        else: 
            if len(v)>0:   
                return v,not isinstance(v[0],StarList)
            return v,True

    def CreateLoop(self,datanames,order=-1):
           """[[datanames]] is a list of datanames that together form a loop.  They should
           all contain the same number of elements (possibly 0). If [[order]] is given, the loop will
           appear at this position in the block. A loop counts as a single position."""
           # check lengths: these datanames should exist
           listed_values = [a for a in datanames if isinstance(self[a],list) and not isinstance(self[a],StarList)]
           if len(listed_values) == len(datanames):
               len_set = set([len(self[a]) for a in datanames])
               if len(len_set)>1:
                   raise ValueError, 'Request to loop datanames %s with different lengths: %s' % (`datanames`,`len_set`)
           elif len(listed_values) != 0:
               raise ValueError, 'Request to loop datanames where some are single values and some are not'
           if len(self.loops)>0:
               loopno = max(self.loops.keys()) + 1
           else:
               loopno = 1
           # store as lower case
           lc_datanames = [d.lower() for d in datanames]
           # remove these datanames from all other loops
           [self.loops[a].remove(b) for a in self.loops for b in lc_datanames if b in self.loops[a]]
           self.loops[loopno] = list(lc_datanames)
           if order >= 0:
               self.item_order.insert(order,loopno)
           else:
               self.item_order.append(loopno)
           # remove these datanames from item ordering
           self.item_order = [a for a in self.item_order if a not in lc_datanames] 
          
    def AddLoopName(self,oldname, newname):
        """Add [[newname]] to the loop containing [[oldname]]"""
        loop_no = self.FindLoop(oldname)
        if loop_no < 0:
            raise KeyError, '%s not in loop' % oldname
        if newname in self.loops[loop_no]:
            return
        # check length
        loop_len = len(self[oldname])
        if len(self[newname]) != loop_len:
            raise ValueError, 'Mismatch of loop column lengths for %s: should be %d' % (newname,loop_len)
        # remove from any other loops
        [self.loops[a].remove(newname) for a in self.loops if newname in self.loops[a]]
        # and add to this loop
        self.loops[loop_no].append(newname)
        
    def FindLoop(self,keyname):
        """Find the loop that contains keyname and return its numerical index,-1 if not present"""
        loop_no = [a for a in self.loops.keys() if keyname.lower() in self.loops[a]]
        if len(loop_no)>0:
            return loop_no[0]
        else:
            return -1

    def GetLoop(self,keyname):
        """Return a LoopBlock object containing keyname"""
        return LoopBlock(self,keyname)

    def GetLoopNames(self,keyname):
        if keyname in self:
            return self.keys()
        for aloop in self.loops:
            try: 
                return aloop.GetLoopNames(keyname)
            except KeyError:
                pass
        raise KeyError, 'Item does not exist'

    def GetLoopNames(self,keyname):
        """Return all datanames appearing together with [[keyname]]"""
        loop_no = self.FindLoop(keyname)
        if loop_no >= 0:
            return self.loops[loop_no]
        else:
            raise KeyError, '%s is not in any loop' % keyname

    def AddLoopName(self,oldname, newname):
        """Add [[newname]] to the loop containing [[oldname]]"""
        loop_no = self.FindLoop(oldname)
        if loop_no < 0:
            raise KeyError, '%s not in loop' % oldname
        if newname in self.loops[loop_no]:
            return
        # check length
        loop_len = len(self[oldname])
        if len(self[newname]) != loop_len:
            raise ValueError, 'Mismatch of loop column lengths for %s: should be %d' % (newname,loop_len)
        # remove from any other loops
        [self.loops[a].remove(newname) for a in self.loops if newname in self.loops[a]]
        # and add to this loop
        self.loops[loop_no].append(newname)
        
    def AddToLoop(self,dataname,loopdata):
        thisloop = self.GetLoop(dataname)
        for itemname,itemvalue in loopdata.items():
            thisloop[itemname] = itemvalue 
        
    def AddToLoop(self,dataname,loopdata):
        # check lengths
        thisloop = self.FindLoop(dataname)
        loop_len = len(self[dataname])
        bad_vals = [a for a in loopdata.items() if len(a[1])!=loop_len]
        if len(bad_vals)>0:
           raise StarLengthError, "Number of values for looped datanames %s not equal to %d" \
               % (`bad_vals`,loop_len)
        self.update(loopdata)
        self.loops[thisloop]+=loopdata.keys()

    def Loopify(self,datanamelist):
        self.CreateLoop(datanamelist)

    def RemoveKeyedPacket(self,keyname,keyvalue):
        packet_coord = list(self[keyname]).index(keyvalue)
        loopnames = self.GetLoopNames(keyname)
        for dataname in loopnames:
            self.block[dataname][0] = list(self.block[dataname][0])
            del self.block[dataname][0][packet_coord]
            self.block[dataname][1] = list(self.block[dataname][1])
            del self.block[dataname][1][packet_coord]

    def GetKeyedPacket(self,keyname,keyvalue,no_case=False):
        """Return the loop packet where [[keyname]] has value [[keyvalue]]. Ignore case if no_case is true"""
        #print "Looking for %s in %s" % (keyvalue, self.parent_block[keyname])
        my_loop = self.GetLoop(keyname)
        if no_case:
           one_pack= [a for a in my_loop if getattr(a,keyname).lower()==keyvalue.lower()]
        else:
           one_pack= [a for a in my_loop if getattr(a,keyname)==keyvalue]
        if len(one_pack)!=1:
            raise ValueError, "Bad packet key %s = %s: returned %d packets" % (keyname,keyvalue,len(one_pack))
        #print "Keyed packet: %s" % one_pack[0]
        return one_pack[0]

    def GetKeyedSemanticPacket(self,keyvalue,cat_id):
        """Return a complete packet for category cat_id"""
        target_keys = self.dictionary.cat_key_table[cat_id]
        p = StarPacket()
        # set case-sensitivity flag
        lcase = False
        if self.dictionary[target_keys[0]]['_type.contents'] in ['Code','Tag','Name']:
            lcase = True
        for cat_key in target_keys:
            try:
                extra_packet = self.GetKeyedPacket(cat_key,keyvalue,no_case=lcase)
            except KeyError:        #try to create the key
                key_vals = self[cat_key]   #will create a key column
            p.merge_packet(self.GetKeyedPacket(cat_key,keyvalue,no_case=lcase))
        # the following attributes used to calculate missing values
        p.key = target_keys[0]
        p.cif_dictionary = self.dictionary
        p.fulldata = self
        return p


    def SetOutputLength(self,wraplength=80,maxoutlength=2048):
        if wraplength > maxoutlength:
            raise StarError("Wrap length (requested %d) must be <= Maximum line length (requested %d)" % (wraplength,maxoutlength))
        self.wraplength = wraplength
        self.maxoutlength = maxoutlength

    def printsection(self,instring='',ordering=[],blockstart="",blockend="",indent=0):
        import string
        # first make an ordering
        self.create_ordering(ordering)
        # now do it...
        if not instring:
            outstring = CIFStringIO(target_width=80)       # the returned string
        else:
            outstring = instring
        # print loop delimiter
        outstring.write(blockstart,canbreak=True)
        while len(self.output_order)>0:
           #print "Remaining to output " + `self.output_order`
           itemname = self.output_order.pop(0)
           item_spec = [i for i in ordering if i['dataname'].lower()==itemname.lower()]
           if len(item_spec)>0:
               col_pos = item_spec[0].get('column',-1)
           else:
               col_pos = -1
               item_spec = {}
               if not isinstance(itemname,int):  #no loop
                   if col_pos < 0: col_pos = 40
                   outstring.set_tab(col_pos)
                   itemvalue = self[itemname]
                   outstring.write(self.true_case[itemname],mustbreak=True,do_tab=False)
                   outstring.write(' ',canbreak=True,do_tab=False)    #space after itemname
                   self.format_value(itemvalue,outstring,hints=item_spec)
               else:   # we are asked to print a loop block
                    #first make sure we have sensible coords.  Length should be one
                    #less than the current dimension
                    outstring.set_tab(10)       #guess this is OK?
                    outstring.write(' '*indent,mustbreak=True,do_tab=False); outstring.write('loop_\n',do_tab=False)
                    self.format_names(outstring,indent+2,loop_no=itemname)
                    self.format_packets(outstring,indent+2,loop_no=itemname)
        else:
            returnstring = outstring.getvalue()
        outstring.close()
        return returnstring

    def format_names(self,outstring,indent=0,loop_no=-1):
        """Print datanames from [[loop_no]] one per line"""
        temp_order = self.loops[loop_no][:]   #copy
        while len(temp_order)>0:
            itemname = temp_order.pop(0)
            outstring.write(' ' * indent,do_tab=False) 
            outstring.write(self.true_case[itemname],do_tab=False)
            outstring.write("\n",do_tab=False)

    def format_packets(self,outstring,indent=0,loop_no=-1):
       import cStringIO
       import string
       alldata = [self[a] for a in self.loops[loop_no]]
       print 'Alldata: %s' % `alldata`
       packet_data = apply(zip,alldata)
       print 'Packet data: %s' % `packet_data`
       for position in range(len(packet_data)):
           for point in range(len(packet_data[position])):
               datapoint = packet_data[position][point]
               packstring = self.format_packet_item(datapoint,indent,outstring)
           outstring.write("\n",do_tab=False)
               
    def format_packet_item(self,pack_item,indent,outstring):
           # print 'Formatting %s' % `pack_item`
           if isinstance(pack_item,basestring):
               outstring.write(self._formatstring(pack_item)) 
           else: 
               self.format_value(pack_item,outstring)
           outstring.write(' ',canbreak=True,do_tab=False)

    def _formatstring(self,instring,delimiter=None,standard='CIF1',indent=0,
        lbprotocol=True,pref_protocol=True):
        import string
        if standard == 'CIF2':
            allowed_delimiters = set(['"',"'",";",None,'"""',"'''"])
        else:
            allowed_delimiters = set(['"',"'",";",None])
        if len(instring)==0: allowed_delimiters.difference_update([None])
        if len(instring) > (self.maxoutlength-2) or '\n' in instring:
                allowed_delimiters.intersection_update([";","'''",'"""'])
        if ' ' in instring or '\t' in instring or '\v' in instring or '_' in instring or ',' in instring:
                allowed_delimiters.difference_update([None])
        if '"' in instring: allowed_delimiters.difference_update(['"',None])
        if "'" in instring: allowed_delimiters.difference_update(["'",None])
        out_delimiter = ";"  #default (most conservative)
        if delimiter in allowed_delimiters:
            out_delimiter = delimiter
        elif "'" in allowed_delimiters: out_delimiter = "'"
        elif '"' in allowed_delimiters: out_delimiter = '"'
        if out_delimiter in ['"',"'",'"""',"'''"]: return out_delimiter + instring + out_delimiter
        elif out_delimiter is None: return instring
        # we are left with semicolon strings
        outstring = "\n;"
        # if there are returns in the string, try to work with them
        while 1:
            retin = string.find(instring,'\n')+1
            if retin < self.maxoutlength and retin > 0:      # honour this break
                outstring = outstring + instring[:retin]
                instring = instring[retin:]
            elif len(instring)0:
               self.format_value(itemvalue[0],stringsink)
               for listval in itemvalue[1:]:
                  print 'Formatting %s' % `listval`
                  stringsink.write(', ',do_tab=False)
                  self.format_value(listval,stringsink,compound=True)
           stringsink.write(']',unindent=True)
        elif isinstance(itemvalue,StarDict):
           stringsink.set_tab(0)
           stringsink.write('{',newindent=True,mustbreak=compound)  #start a new line inside
           items = itemvalue.items()
           if len(items)>0:
               stringsink.write("'"+items[0][0]+"'"+':',canbreak=True)
               self.format_value(items[0][1],stringsink)
               for key,value in items[1:]:
                   stringsink.write(', ')
                   stringsink.write("'"+key+"'"+":",canbreak=True)
                   self.format_value(value,stringsink)   #never break between key and value
           stringsink.write('}',unindent=True)
        else: 
           stringsink.write(str(itemvalue),canbreak=True)   #numbers

    def process_template(self,template_string):
        """Process a template datafile to formatting instructions"""
        template_as_cif = StarFile(StringIO(template_string),grammar="DDLm").first_block()
        #template_as_lines = template_string.split("\n")
        #template_as_lines = [l for l in template_as_lines if len(l)>0 and l[0]!='#']
        #template_as_lines = [l for l in template_as_lines if l.split()[0] != 'loop_']
        #template_full_lines = dict([(l.split()[0],l) for l in template_as_lines if len(l.split())>0])
        self.form_hints = []   #ordered array of hint dictionaries
        for item in template_as_cif.item_order:  #order of input
            if not isinstance(item,int):    #not nested
                hint_dict = {"dataname":item}
                # find the line in the file
                start_pos = re.search("(^[ \t]*" + item + "[ \t\n]+)(?P([\S]+)|(^;))",template_string,re.I|re.M)
                if start_pos.group("spec") != None:
                    spec_pos = start_pos.start("spec")-start_pos.start(0)
                    spec_char = template_string[start_pos.start("spec")]
                    if spec_char in '\'";':
                        hint_dict.update({"delimiter":spec_char})
                    if spec_char != ";":   #so we need to work out the column number
                        hint_dict.update({"column":spec_pos})
                print '%s: %s' % (item,`hint_dict`)
                self.form_hints.append(hint_dict)
            else:           #loop block
                testnames = template_as_cif.loops[item]
                total_items = len(template_as_cif.loops[item])
                testname = testnames[0]
                #find the loop spec line in the file
                loop_regex = "(^[ \t]*loop_[ \t\n\r]+" + testname + "([ \t\n\r]+_[\S]+){%d}[ \t]*$(?P(.(?!_loop|_[\S]+))*))" % (total_items - 1)
                loop_line = re.search(loop_regex,template_string,re.I|re.M|re.S)
                loop_so_far = loop_line.end()
                packet_text = loop_line.group('packet')
                packet_regex = "[ \t]*(?P(?P'([^\n\r\f']*)'+)|(?P\"([^\n\r\"]*)\"+)|(?P[^\s]+))"
                packet_pos = re.finditer(packet_regex,packet_text)
                line_end_pos = re.finditer("^",packet_text,re.M)
                next_end = line_end_pos.next().end()
                last_end = next_end
                for loopname in testnames:
                    hint_dict = {"dataname":loopname}
                    thismatch = packet_pos.next()
                    while thismatch.start('all') > next_end:
                        try: 
                            last_end = next_end
                            next_end = line_end_pos.next().start()
                            print 'next end %d' % next_end
                        except StopIteration:
                            pass 
                    print 'Start %d, last_end %d' % (thismatch.start('all'),last_end)
                    col_pos = thismatch.start('all') - last_end
                    if thismatch.group('none') is None:
                        hint_dict.update({'delimiter':thismatch.groups()[0][0]})
                    hint_dict.update({'column':col_pos})
                    print '%s: %s' % (loopname,`hint_dict`)
                    self.form_hints.append(hint_dict)
        return

    def create_ordering(self,order_dict):
        """Create a canonical ordering that includes loops using our formatting hints dictionary"""
        requested_order = [i['dataname'] for i in order_dict]
        new_order = []
        for item in requested_order:
           if isinstance(item,basestring) and item.lower() in self.item_order:
               new_order.append(item.lower())
           elif self.has_key(item):    #in a loop somewhere
               target_loop = self.FindLoop(item)
               if target_loop not in new_order:
                   new_order.append(target_loop)
        extras = [i for i in self.item_order if i not in new_order]
        self.output_order = new_order + extras
        print 'Final order: ' + `self.output_order`

    def convert_to_string(self,dataname):
        """Convert values held in dataname value fork to string version"""
        v,is_value = self.GetFullItemValue(dataname)
        if not is_value:
            return v
        if check_stringiness(v): return v   #already strings
        # TODO...something else
        return v


    def merge(self,new_block,mode="strict",match_att=[],match_function=None,
                   rel_keys = []):
        if mode == 'strict':
           for key in new_block.keys(): 
               if self.has_key(key) and key not in match_att:
                  raise CifError( "Identical keys %s in strict merge mode" % key)
               elif key not in match_att:           #a new dataname
                   self[key] = new_block[key]
           # we get here if there are no keys in common, so we can now copy
           # the loops and not worry about overlaps
           for one_loop in new_block.loops.values():
               self.CreateLoop(one_loop)
           # we have lost case information
           self.true_case.update(new_block.true_case)
        elif mode == 'replace':
           newkeys = new_block.keys()
           for ma in match_att:
              try:
                   newkeys.remove(ma)        #don't touch the special ones
              except ValueError:
                   pass
           for key in new_block.keys(): 
                  if isinstance(key,basestring):
                      self[key] = new_block[key] 
           # creating the loop will remove items from other loops
           for one_loop in new_block.loops.values():
               self.CreateLoop(one_loop)
           # we have lost case information
           self.true_case.update(new_block.true_case)
        elif mode == 'overlay':
           print 'Overlay mode, current overwrite is %s' % self.overwrite
           raise StarError, 'Overlay block merge mode not implemented'
           save_overwrite = self.overwrite
           self.overwrite = True
           for attribute in new_block.keys():
               if attribute in match_att: continue      #ignore this one
               new_value = new_block[attribute]
               #non-looped items
               if new_block.FindLoop(attribute)<0:     #not looped
                  self[attribute] = new_value 
           my_loops = self.loops.values()
           perfect_overlaps = [a for a in new_block.loops if a in my_loops]
           for po in perfect_overlaps:
              loop_keys = [a for a in po if a in rel_keys]  #do we have a key?
              try:
                  newkeypos = map(lambda a:newkeys.index(a),loop_keys)
                  newkeypos = newkeypos[0]      #one key per loop for now
                  loop_keys = loop_keys[0] 
              except (ValueError,IndexError):
                  newkeypos = []
                  overlap_data = map(lambda a:listify(self[a]),overlaps) #old packet data
                  new_data = map(lambda a:new_block[a],overlaps) #new packet data
                  packet_data = transpose(overlap_data)
                  new_p_data = transpose(new_data)
                  # remove any packets for which the keys match between old and new; we
                  # make the arbitrary choice that the old data stays
                  if newkeypos:
                      # get matching values in new list
                      print "Old, new data:\n%s\n%s" % (`overlap_data[newkeypos]`,`new_data[newkeypos]`)
                      key_matches = filter(lambda a:a in overlap_data[newkeypos],new_data[newkeypos])
                      # filter out any new data with these key values
                      new_p_data = filter(lambda a:a[newkeypos] not in key_matches,new_p_data)
                      if new_p_data:
                          new_data = transpose(new_p_data)
                      else: new_data = []
                  # wipe out the old data and enter the new stuff
                  byebyeloop = self.GetLoop(overlaps[0])
                  # print "Removing '%s' with overlaps '%s'" % (`byebyeloop`,`overlaps`)
                  # Note that if, in the original dictionary, overlaps are not
                  # looped, GetLoop will return the block itself.  So we check
                  # for this case...
                  if byebyeloop != self:
                      self.remove_loop(byebyeloop)
                  self.AddLoopItem((overlaps,overlap_data))  #adding old packets
                  for pd in new_p_data:                             #adding new packets
                     if pd not in packet_data:
                        for i in range(len(overlaps)):
                            #don't do this at home; we are appending
                            #to something in place
                            self[overlaps[i]].append(pd[i]) 
           self.overwrite = save_overwrite

    def assign_dictionary(self,dic):
        if not dic.diclang=="DDLm":
            print "Warning: ignoring dictionary %s" % dic.dic_as_cif.my_uri
            return
        self.dictionary = dic

    def unassign_dictionary(self):
        """Remove dictionary-dependent behaviour"""
        self.dictionary = None

Ancestors (in MRO)

Instance variables

var block

var characterset

var curitem

var dictionary

var formatting_hints

var item_order

var loops

var maxoutlength

var overwrite

var popout

var provide_value

var true_case

var wraplength

Methods

def __init__(

self, data=(), maxoutlength=2048, wraplength=80, overwrite=True, characterset='ascii', maxnamelength=-1)

def __init__(self,data = (), maxoutlength=2048, wraplength=80, overwrite=True,
             characterset='ascii',maxnamelength=-1):
    self.block = {}    #the actual data storage (lower case keys)
    self.loops = {}    #each loop is indexed by a number and contains a list of datanames
    self.item_order = []  #lower case, loops referenced by integer
    self.formatting_hints = {}
    self.true_case = {} #transform lower case to supplied case
    self.provide_value = False   #prefer string version always
    self.dictionary = None      #DDLm dictionary
    self.popout = False         #used during load iteration
    self.curitem = -1           #used during iteration
    self.maxoutlength = maxoutlength
    self.setmaxnamelength(maxnamelength)  #to enforce CIF limit of 75 characters
    self.wraplength = wraplength
    self.overwrite = overwrite
    self.characterset = characterset
    if self.characterset == 'ascii':
        self.char_check = re.compile("[][ \n\r\t!%&\(\)*+,./:<=>?@0-9A-Za-z\\\\^`{}\|~\"#$';_-]+",re.M)
    elif self.characterset == 'unicode':
        self.char_check = re.compile(u"[][ \n\r\t!%&\(\)*+,./:<=>?@0-9A-Za-z\\\\^`{}\|~\"#$';_\u00A0-\uD7FF\uE000-\uFDCF\uFDF0-\uFFFD\U00010000-\U0010FFFD-]+",re.M)
    else:
        raise StarError("No character set specified")
    if isinstance(data,(tuple,list)):
        for item in data:
            self.AddLoopItem(item)
    elif isinstance(data,StarBlock):
        self.block = data.block.copy() 
        self.item_order = data.item_order[:]
        self.true_case = data.true_case.copy()
        # loops as well
        self.loops = data.loops.copy()

def AddItem(

self, key, value, precheck=False)

def AddItem(self,key,value,precheck=False):
    if not isinstance(key,basestring):
         raise TypeError, 'Star datanames are strings only (got %s)' % `key`
    if not precheck:
         self.check_data_name(key,self.maxnamelength)    # make sure no nasty characters   
    # check for overwriting
    if self.has_key(key):
         if not self.overwrite:
             raise StarError( 'Attempt to insert duplicate item name %s' % key)
    # put the data in the right container
    regval,empty_val = self.regularise_data(value)
    # check for pure string data
    pure_string = check_stringiness(regval)
    if not precheck:
        self.check_item_value(regval)
    # update ancillary information first
    lower_key = key.lower()
    if not self.has_key(lower_key) and self.FindLoop(lower_key)<0:      #need to add to order
        self.item_order.append(lower_key)
    # always remove from our case table in case the case is different
    else:
        del self.true_case[lower_key]
    self.true_case[lower_key] = key
    if pure_string:
        self.block.update({lower_key:[regval,empty_val]})
    else:
        self.block.update({lower_key:[empty_val,regval]})  

def AddLoopItem(

self, incomingdata, precheck=False, maxlength=-1)

def AddLoopItem(self,incomingdata,precheck=False,maxlength=-1):
    # print "Received data %s" % `incomingdata`
    # we accept tuples, strings, lists and dicts!!
    # Direct insertion: we have a string-valued key, with an array
    # of values -> single-item into our loop
    if isinstance(incomingdata[0],(tuple,list)):
       # a whole loop
       keyvallist = zip(incomingdata[0],incomingdata[1])
       for key,value in keyvallist:
           self.AddItem(key,value)
       self.CreateLoop(incomingdata[0])
    elif not isinstance(incomingdata[0],basestring):
         raise TypeError, 'Star datanames are strings only (got %s)' % `incomingdata[0]`
    else:
        self.AddItem(incomingdata[0],incomingdata[1])

def AddLoopName(

self, oldname, newname)

Add [[newname]] to the loop containing [[oldname]]

def AddLoopName(self,oldname, newname):
    """Add [[newname]] to the loop containing [[oldname]]"""
    loop_no = self.FindLoop(oldname)
    if loop_no < 0:
        raise KeyError, '%s not in loop' % oldname
    if newname in self.loops[loop_no]:
        return
    # check length
    loop_len = len(self[oldname])
    if len(self[newname]) != loop_len:
        raise ValueError, 'Mismatch of loop column lengths for %s: should be %d' % (newname,loop_len)
    # remove from any other loops
    [self.loops[a].remove(newname) for a in self.loops if newname in self.loops[a]]
    # and add to this loop
    self.loops[loop_no].append(newname)

def AddToLoop(

self, dataname, loopdata)

def AddToLoop(self,dataname,loopdata):
    # check lengths
    thisloop = self.FindLoop(dataname)
    loop_len = len(self[dataname])
    bad_vals = [a for a in loopdata.items() if len(a[1])!=loop_len]
    if len(bad_vals)>0:
       raise StarLengthError, "Number of values for looped datanames %s not equal to %d" \
           % (`bad_vals`,loop_len)
    self.update(loopdata)
    self.loops[thisloop]+=loopdata.keys()

def ChangeItemOrder(

self, itemname, newpos)

Move itemname to newpos

def ChangeItemOrder(self,itemname,newpos):
    """Move itemname to newpos"""
    if isinstance(itemname,basestring):
        true_name = itemname.lower()
    else:
        true_name = itemname
    self.item_order.remove(true_name)
    self.item_order.insert(newpos,true_name)

def CreateLoop(

self, datanames, order=-1)

[[datanames]] is a list of datanames that together form a loop. They should all contain the same number of elements (possibly 0). If [[order]] is given, the loop will appear at this position in the block. A loop counts as a single position.

def CreateLoop(self,datanames,order=-1):
       """[[datanames]] is a list of datanames that together form a loop.  They should
       all contain the same number of elements (possibly 0). If [[order]] is given, the loop will
       appear at this position in the block. A loop counts as a single position."""
       # check lengths: these datanames should exist
       listed_values = [a for a in datanames if isinstance(self[a],list) and not isinstance(self[a],StarList)]
       if len(listed_values) == len(datanames):
           len_set = set([len(self[a]) for a in datanames])
           if len(len_set)>1:
               raise ValueError, 'Request to loop datanames %s with different lengths: %s' % (`datanames`,`len_set`)
       elif len(listed_values) != 0:
           raise ValueError, 'Request to loop datanames where some are single values and some are not'
       if len(self.loops)>0:
           loopno = max(self.loops.keys()) + 1
       else:
           loopno = 1
       # store as lower case
       lc_datanames = [d.lower() for d in datanames]
       # remove these datanames from all other loops
       [self.loops[a].remove(b) for a in self.loops for b in lc_datanames if b in self.loops[a]]
       self.loops[loopno] = list(lc_datanames)
       if order >= 0:
           self.item_order.insert(order,loopno)
       else:
           self.item_order.append(loopno)
       # remove these datanames from item ordering
       self.item_order = [a for a in self.item_order if a not in lc_datanames] 

def FindLoop(

self, keyname)

Find the loop that contains keyname and return its numerical index,-1 if not present

def FindLoop(self,keyname):
    """Find the loop that contains keyname and return its numerical index,-1 if not present"""
    loop_no = [a for a in self.loops.keys() if keyname.lower() in self.loops[a]]
    if len(loop_no)>0:
        return loop_no[0]
    else:
        return -1

def GetFullItemValue(

self, itemname)

Return value of itemname and whether or not it is a native value

def GetFullItemValue(self,itemname):
    """Return value of itemname and whether or not it is a native value"""
    try:
        s,v = self.block[itemname.lower()]
    except KeyError:
        raise KeyError, 'Itemname %s not in datablock' % itemname
    # prefer string value unless all are None
    # are we a looped value?
    if not isinstance(s,(tuple,list)) or isinstance(s,StarList):
        if s is not None or (isinstance(s,StarList) and None not in s): 
            return s,False    #a string value
        else:
            return v,not isinstance(v,StarList)  #a StarList is not calculation-ready
    elif None not in s: 
        return s,False     #a list of string values
    else: 
        if len(v)>0:   
            return v,not isinstance(v[0],StarList)
        return v,True

def GetItemOrder(

self)

def GetItemOrder(self):
    return self.item_order[:]

def GetItemPosition(

self, itemname)

def GetItemPosition(self,itemname):
    import string
    if isinstance(itemname,int):
        # return loop position
        return (-1, self.item_order.index(itemname))
    if not self.has_key(itemname):
        raise ValueError, 'No such dataname %s' % itemname
    testname = itemname.lower()
    if testname in self.item_order:
        return (-1,self.item_order.index(testname))
    loop_no = self.FindLoop(testname)
    loop_pos = self.loops[loop_no].index(testname)
    return loop_no,loop_pos

def GetItemValue(

self, itemname)

Return value of itemname

def GetItemValue(self,itemname):
    """Return value of itemname"""
    return self.GetFullItemValue(itemname)[0]

def GetKeyedPacket(

self, keyname, keyvalue, no_case=False)

Return the loop packet where [[keyname]] has value [[keyvalue]]. Ignore case if no_case is true

def GetKeyedPacket(self,keyname,keyvalue,no_case=False):
    """Return the loop packet where [[keyname]] has value [[keyvalue]]. Ignore case if no_case is true"""
    #print "Looking for %s in %s" % (keyvalue, self.parent_block[keyname])
    my_loop = self.GetLoop(keyname)
    if no_case:
       one_pack= [a for a in my_loop if getattr(a,keyname).lower()==keyvalue.lower()]
    else:
       one_pack= [a for a in my_loop if getattr(a,keyname)==keyvalue]
    if len(one_pack)!=1:
        raise ValueError, "Bad packet key %s = %s: returned %d packets" % (keyname,keyvalue,len(one_pack))
    #print "Keyed packet: %s" % one_pack[0]
    return one_pack[0]

def GetKeyedSemanticPacket(

self, keyvalue, cat_id)

Return a complete packet for category cat_id

def GetKeyedSemanticPacket(self,keyvalue,cat_id):
    """Return a complete packet for category cat_id"""
    target_keys = self.dictionary.cat_key_table[cat_id]
    p = StarPacket()
    # set case-sensitivity flag
    lcase = False
    if self.dictionary[target_keys[0]]['_type.contents'] in ['Code','Tag','Name']:
        lcase = True
    for cat_key in target_keys:
        try:
            extra_packet = self.GetKeyedPacket(cat_key,keyvalue,no_case=lcase)
        except KeyError:        #try to create the key
            key_vals = self[cat_key]   #will create a key column
        p.merge_packet(self.GetKeyedPacket(cat_key,keyvalue,no_case=lcase))
    # the following attributes used to calculate missing values
    p.key = target_keys[0]
    p.cif_dictionary = self.dictionary
    p.fulldata = self
    return p

def GetLoop(

self, keyname)

Return a LoopBlock object containing keyname

def GetLoop(self,keyname):
    """Return a LoopBlock object containing keyname"""
    return LoopBlock(self,keyname)

def GetLoopNames(

self, keyname)

Return all datanames appearing together with [[keyname]]

def GetLoopNames(self,keyname):
    """Return all datanames appearing together with [[keyname]]"""
    loop_no = self.FindLoop(keyname)
    if loop_no >= 0:
        return self.loops[loop_no]
    else:
        raise KeyError, '%s is not in any loop' % keyname

def Loopify(

self, datanamelist)

def Loopify(self,datanamelist):
    self.CreateLoop(datanamelist)

def RemoveItem(

self, itemname)

def RemoveItem(self,itemname):
    # first check any loops
    loop_no = self.FindLoop(itemname)
    testkey = itemname.lower()
    if self.has_key(testkey):
        del self.block[testkey]
        del self.true_case[testkey]
        # now remove from loop
        if loop_no >= 0:
            self.loops[loop_no].remove(testkey)
            if len(self.loops[loop_no])==0:
                del self.loops[loop_no]
                self.item_order.remove(loop_no)
        else:  #will appear in order list
            self.item_order.remove(testkey)

def RemoveKeyedPacket(

self, keyname, keyvalue)

def RemoveKeyedPacket(self,keyname,keyvalue):
    packet_coord = list(self[keyname]).index(keyvalue)
    loopnames = self.GetLoopNames(keyname)
    for dataname in loopnames:
        self.block[dataname][0] = list(self.block[dataname][0])
        del self.block[dataname][0][packet_coord]
        self.block[dataname][1] = list(self.block[dataname][1])
        del self.block[dataname][1][packet_coord]

def RemoveLoopItem(

self, itemname)

def RemoveLoopItem(self,itemname):
    self.RemoveItem(itemname)

def SetOutputLength(

self, wraplength=80, maxoutlength=2048)

def SetOutputLength(self,wraplength=80,maxoutlength=2048):
    if wraplength > maxoutlength:
        raise StarError("Wrap length (requested %d) must be <= Maximum line length (requested %d)" % (wraplength,maxoutlength))
    self.wraplength = wraplength
    self.maxoutlength = maxoutlength

def assign_dictionary(

self, dic)

def assign_dictionary(self,dic):
    if not dic.diclang=="DDLm":
        print "Warning: ignoring dictionary %s" % dic.dic_as_cif.my_uri
        return
    self.dictionary = dic

def check_data_name(

self, dataname, maxlength=-1)

def check_data_name(self,dataname,maxlength=-1): 
    if maxlength > 0:
        self.check_name_length(dataname,maxlength)
    if dataname[0]!='_':
        raise StarError( 'Dataname ' + dataname + ' does not begin with _')
    if self.characterset=='ascii':
        if len (filter (lambda a: ord(a) < 33 or ord(a) > 126, dataname)) > 0:
            raise StarError( 'Dataname ' + dataname + ' contains forbidden characters')
    else:
        # print 'Checking %s for unicode characterset conformance' % dataname
        if len (filter (lambda a: ord(a) < 33, dataname)) > 0:
            raise StarError( 'Dataname ' + dataname + ' contains forbidden characters (below code point 33)')
        if len (filter (lambda a: ord(a) > 126 and ord(a) < 160, dataname)) > 0:
            raise StarError( 'Dataname ' + dataname + ' contains forbidden characters (between code point 127-159)')
        if len (filter (lambda a: ord(a) > 0xD7FF and ord(a) < 0xE000, dataname)) > 0:
            raise StarError( 'Dataname ' + dataname + ' contains unsupported characters (between U+D800 and U+E000)')
        if len (filter (lambda a: ord(a) > 0xFDCF and ord(a) < 0xFDF0, dataname)) > 0:
            raise StarError( 'Dataname ' + dataname + ' contains unsupported characters (between U+FDD0 and U+FDEF)')
        if len (filter (lambda a: ord(a) == 0xFFFE or ord(a) == 0xFFFF, dataname)) > 0:
            raise StarError( 'Dataname ' + dataname + ' contains unsupported characters (U+FFFE and/or U+FFFF)')
        if len (filter (lambda a: ord(a) > 0x10000 and (ord(a) & 0xE == 0xE) , dataname)) > 0:
            print '%s fails' % dataname
            for a in dataname: print '%x' % ord(a),
            print
            raise StarError( u'Dataname ' + dataname + u' contains unsupported characters (U+xFFFE and/or U+xFFFF)')

def check_item_value(

self, item)

def check_item_value(self,item):
    test_item = item
    if not isinstance(item,(list,dict,tuple)):
       test_item = [item]         #single item list
    def check_one (it):
        if isinstance(it,basestring):
            if it=='': return
            me = self.char_check.match(it)            
            if not me:
                print "Fail value check: %s" % it
                raise StarError, u'Bad character in %s' % it
            else:
                if me.span() != (0,len(it)):
                    print "Fail value check, match only %d-%d in string %s" % (me.span()[0],me.span()[1],`it`)
                    raise StarError,u'Data item "' + `it` +  u'"... contains forbidden characters'
    map(check_one,test_item)

def check_name_length(

self, dataname, maxlength)

def check_name_length(self,dataname,maxlength):
    if len(dataname)>maxlength:
        raise StarError( 'Dataname %s exceeds maximum length %d' % (dataname,maxlength))
    return

def clear(

self)

def clear(self):
    self.block = {}
    self.loops = {}
    self.item_order = []
    self.true_case = {}

def convert_to_string(

self, dataname)

Convert values held in dataname value fork to string version

def convert_to_string(self,dataname):
    """Convert values held in dataname value fork to string version"""
    v,is_value = self.GetFullItemValue(dataname)
    if not is_value:
        return v
    if check_stringiness(v): return v   #already strings
    # TODO...something else
    return v

def copy(

self)

def copy(self):
    newcopy = StarBlock()
    newcopy.block = self.block.copy()
    newcopy.loops = []
    newcopy.item_order = self.item_order[:]
    newcopy.true_case = self.true_case.copy()
    newcopy.loops = self.loops.copy()
#    return self.copy.im_class(newcopy)   #catch inheritance
    return newcopy

def create_ordering(

self, order_dict)

Create a canonical ordering that includes loops using our formatting hints dictionary

def create_ordering(self,order_dict):
    """Create a canonical ordering that includes loops using our formatting hints dictionary"""
    requested_order = [i['dataname'] for i in order_dict]
    new_order = []
    for item in requested_order:
       if isinstance(item,basestring) and item.lower() in self.item_order:
           new_order.append(item.lower())
       elif self.has_key(item):    #in a loop somewhere
           target_loop = self.FindLoop(item)
           if target_loop not in new_order:
               new_order.append(target_loop)
    extras = [i for i in self.item_order if i not in new_order]
    self.output_order = new_order + extras
    print 'Final order: ' + `self.output_order`

def format_names(

self, outstring, indent=0, loop_no=-1)

Print datanames from [[loop_no]] one per line

def format_names(self,outstring,indent=0,loop_no=-1):
    """Print datanames from [[loop_no]] one per line"""
    temp_order = self.loops[loop_no][:]   #copy
    while len(temp_order)>0:
        itemname = temp_order.pop(0)
        outstring.write(' ' * indent,do_tab=False) 
        outstring.write(self.true_case[itemname],do_tab=False)
        outstring.write("\n",do_tab=False)

def format_packet_item(

self, pack_item, indent, outstring)

def format_packet_item(self,pack_item,indent,outstring):
       # print 'Formatting %s' % `pack_item`
       if isinstance(pack_item,basestring):
           outstring.write(self._formatstring(pack_item)) 
       else: 
           self.format_value(pack_item,outstring)
       outstring.write(' ',canbreak=True,do_tab=False)

def format_packets(

self, outstring, indent=0, loop_no=-1)

def format_packets(self,outstring,indent=0,loop_no=-1):
   import cStringIO
   import string
   alldata = [self[a] for a in self.loops[loop_no]]
   print 'Alldata: %s' % `alldata`
   packet_data = apply(zip,alldata)
   print 'Packet data: %s' % `packet_data`
   for position in range(len(packet_data)):
       for point in range(len(packet_data[position])):
           datapoint = packet_data[position][point]
           packstring = self.format_packet_item(datapoint,indent,outstring)
       outstring.write("\n",do_tab=False)

def format_value(

self, itemvalue, stringsink, compound=False, hints={})

Format a Star data value

def format_value(self,itemvalue,stringsink,compound=False,hints={}):
    """Format a Star data value"""
    delimiter = hints.get('delimiter',None)
    if isinstance(itemvalue,basestring):  #need to sanitize
       stringsink.write(self._formatstring(itemvalue,delimiter=delimiter),canbreak = True)
    elif isinstance(itemvalue,StarList):
       stringsink.set_tab(0)
       stringsink.write('[',canbreak=True,newindent=True,mustbreak=compound)
       if len(itemvalue)>0:
           self.format_value(itemvalue[0],stringsink)
           for listval in itemvalue[1:]:
              print 'Formatting %s' % `listval`
              stringsink.write(', ',do_tab=False)
              self.format_value(listval,stringsink,compound=True)
       stringsink.write(']',unindent=True)
    elif isinstance(itemvalue,StarDict):
       stringsink.set_tab(0)
       stringsink.write('{',newindent=True,mustbreak=compound)  #start a new line inside
       items = itemvalue.items()
       if len(items)>0:
           stringsink.write("'"+items[0][0]+"'"+':',canbreak=True)
           self.format_value(items[0][1],stringsink)
           for key,value in items[1:]:
               stringsink.write(', ')
               stringsink.write("'"+key+"'"+":",canbreak=True)
               self.format_value(value,stringsink)   #never break between key and value
       stringsink.write('}',unindent=True)
    else: 
       stringsink.write(str(itemvalue),canbreak=True)   #numbers

def get(

self, key, default=None)

def get(self,key,default=None):
    if self.has_key(key):
        retval = self.__getitem__(key)
    else:
        retval = default
    return retval

def has_key(

self, key)

def has_key(self,key):
    if isinstance(key,basestring) and key.lower() in self.keys():
        return 1
    return 0

def items(

self)

def items(self):
    return [a for a in zip(self.keys(),self.values())]

def keys(

self)

def keys(self):
    return self.block.keys()    #always lower case

def merge(

self, new_block, mode='strict', match_att=[], match_function=None, rel_keys=[])

def merge(self,new_block,mode="strict",match_att=[],match_function=None,
               rel_keys = []):
    if mode == 'strict':
       for key in new_block.keys(): 
           if self.has_key(key) and key not in match_att:
              raise CifError( "Identical keys %s in strict merge mode" % key)
           elif key not in match_att:           #a new dataname
               self[key] = new_block[key]
       # we get here if there are no keys in common, so we can now copy
       # the loops and not worry about overlaps
       for one_loop in new_block.loops.values():
           self.CreateLoop(one_loop)
       # we have lost case information
       self.true_case.update(new_block.true_case)
    elif mode == 'replace':
       newkeys = new_block.keys()
       for ma in match_att:
          try:
               newkeys.remove(ma)        #don't touch the special ones
          except ValueError:
               pass
       for key in new_block.keys(): 
              if isinstance(key,basestring):
                  self[key] = new_block[key] 
       # creating the loop will remove items from other loops
       for one_loop in new_block.loops.values():
           self.CreateLoop(one_loop)
       # we have lost case information
       self.true_case.update(new_block.true_case)
    elif mode == 'overlay':
       print 'Overlay mode, current overwrite is %s' % self.overwrite
       raise StarError, 'Overlay block merge mode not implemented'
       save_overwrite = self.overwrite
       self.overwrite = True
       for attribute in new_block.keys():
           if attribute in match_att: continue      #ignore this one
           new_value = new_block[attribute]
           #non-looped items
           if new_block.FindLoop(attribute)<0:     #not looped
              self[attribute] = new_value 
       my_loops = self.loops.values()
       perfect_overlaps = [a for a in new_block.loops if a in my_loops]
       for po in perfect_overlaps:
          loop_keys = [a for a in po if a in rel_keys]  #do we have a key?
          try:
              newkeypos = map(lambda a:newkeys.index(a),loop_keys)
              newkeypos = newkeypos[0]      #one key per loop for now
              loop_keys = loop_keys[0] 
          except (ValueError,IndexError):
              newkeypos = []
              overlap_data = map(lambda a:listify(self[a]),overlaps) #old packet data
              new_data = map(lambda a:new_block[a],overlaps) #new packet data
              packet_data = transpose(overlap_data)
              new_p_data = transpose(new_data)
              # remove any packets for which the keys match between old and new; we
              # make the arbitrary choice that the old data stays
              if newkeypos:
                  # get matching values in new list
                  print "Old, new data:\n%s\n%s" % (`overlap_data[newkeypos]`,`new_data[newkeypos]`)
                  key_matches = filter(lambda a:a in overlap_data[newkeypos],new_data[newkeypos])
                  # filter out any new data with these key values
                  new_p_data = filter(lambda a:a[newkeypos] not in key_matches,new_p_data)
                  if new_p_data:
                      new_data = transpose(new_p_data)
                  else: new_data = []
              # wipe out the old data and enter the new stuff
              byebyeloop = self.GetLoop(overlaps[0])
              # print "Removing '%s' with overlaps '%s'" % (`byebyeloop`,`overlaps`)
              # Note that if, in the original dictionary, overlaps are not
              # looped, GetLoop will return the block itself.  So we check
              # for this case...
              if byebyeloop != self:
                  self.remove_loop(byebyeloop)
              self.AddLoopItem((overlaps,overlap_data))  #adding old packets
              for pd in new_p_data:                             #adding new packets
                 if pd not in packet_data:
                    for i in range(len(overlaps)):
                        #don't do this at home; we are appending
                        #to something in place
                        self[overlaps[i]].append(pd[i]) 
       self.overwrite = save_overwrite

def printsection(

self, instring='', ordering=[], blockstart='', blockend='', indent=0)

def printsection(self,instring='',ordering=[],blockstart="",blockend="",indent=0):
    import string
    # first make an ordering
    self.create_ordering(ordering)
    # now do it...
    if not instring:
        outstring = CIFStringIO(target_width=80)       # the returned string
    else:
        outstring = instring
    # print loop delimiter
    outstring.write(blockstart,canbreak=True)
    while len(self.output_order)>0:
       #print "Remaining to output " + `self.output_order`
       itemname = self.output_order.pop(0)
       item_spec = [i for i in ordering if i['dataname'].lower()==itemname.lower()]
       if len(item_spec)>0:
           col_pos = item_spec[0].get('column',-1)
       else:
           col_pos = -1
           item_spec = {}
           if not isinstance(itemname,int):  #no loop
               if col_pos < 0: col_pos = 40
               outstring.set_tab(col_pos)
               itemvalue = self[itemname]
               outstring.write(self.true_case[itemname],mustbreak=True,do_tab=False)
               outstring.write(' ',canbreak=True,do_tab=False)    #space after itemname
               self.format_value(itemvalue,outstring,hints=item_spec)
           else:   # we are asked to print a loop block
                #first make sure we have sensible coords.  Length should be one
                #less than the current dimension
                outstring.set_tab(10)       #guess this is OK?
                outstring.write(' '*indent,mustbreak=True,do_tab=False); outstring.write('loop_\n',do_tab=False)
                self.format_names(outstring,indent+2,loop_no=itemname)
                self.format_packets(outstring,indent+2,loop_no=itemname)
    else:
        returnstring = outstring.getvalue()
    outstring.close()
    return returnstring

def process_template(

self, template_string)

Process a template datafile to formatting instructions

def process_template(self,template_string):
    """Process a template datafile to formatting instructions"""
    template_as_cif = StarFile(StringIO(template_string),grammar="DDLm").first_block()
    #template_as_lines = template_string.split("\n")
    #template_as_lines = [l for l in template_as_lines if len(l)>0 and l[0]!='#']
    #template_as_lines = [l for l in template_as_lines if l.split()[0] != 'loop_']
    #template_full_lines = dict([(l.split()[0],l) for l in template_as_lines if len(l.split())>0])
    self.form_hints = []   #ordered array of hint dictionaries
    for item in template_as_cif.item_order:  #order of input
        if not isinstance(item,int):    #not nested
            hint_dict = {"dataname":item}
            # find the line in the file
            start_pos = re.search("(^[ \t]*" + item + "[ \t\n]+)(?P([\S]+)|(^;))",template_string,re.I|re.M)
            if start_pos.group("spec") != None:
                spec_pos = start_pos.start("spec")-start_pos.start(0)
                spec_char = template_string[start_pos.start("spec")]
                if spec_char in '\'";':
                    hint_dict.update({"delimiter":spec_char})
                if spec_char != ";":   #so we need to work out the column number
                    hint_dict.update({"column":spec_pos})
            print '%s: %s' % (item,`hint_dict`)
            self.form_hints.append(hint_dict)
        else:           #loop block
            testnames = template_as_cif.loops[item]
            total_items = len(template_as_cif.loops[item])
            testname = testnames[0]
            #find the loop spec line in the file
            loop_regex = "(^[ \t]*loop_[ \t\n\r]+" + testname + "([ \t\n\r]+_[\S]+){%d}[ \t]*$(?P(.(?!_loop|_[\S]+))*))" % (total_items - 1)
            loop_line = re.search(loop_regex,template_string,re.I|re.M|re.S)
            loop_so_far = loop_line.end()
            packet_text = loop_line.group('packet')
            packet_regex = "[ \t]*(?P(?P'([^\n\r\f']*)'+)|(?P\"([^\n\r\"]*)\"+)|(?P[^\s]+))"
            packet_pos = re.finditer(packet_regex,packet_text)
            line_end_pos = re.finditer("^",packet_text,re.M)
            next_end = line_end_pos.next().end()
            last_end = next_end
            for loopname in testnames:
                hint_dict = {"dataname":loopname}
                thismatch = packet_pos.next()
                while thismatch.start('all') > next_end:
                    try: 
                        last_end = next_end
                        next_end = line_end_pos.next().start()
                        print 'next end %d' % next_end
                    except StopIteration:
                        pass 
                print 'Start %d, last_end %d' % (thismatch.start('all'),last_end)
                col_pos = thismatch.start('all') - last_end
                if thismatch.group('none') is None:
                    hint_dict.update({'delimiter':thismatch.groups()[0][0]})
                hint_dict.update({'column':col_pos})
                print '%s: %s' % (loopname,`hint_dict`)
                self.form_hints.append(hint_dict)
    return

def regularise_data(

self, dataitem)

Place dataitem into a list if necessary

def regularise_data(self,dataitem):
    """Place dataitem into a list if necessary"""
    from numbers import Number
    if isinstance(dataitem,(Number,basestring,StarList,StarDict)):
        return dataitem,None
    if isinstance(dataitem,(tuple,list)):
        return dataitem,[None]*len(dataitem)
    # so try to make into a list
    try:
        regval = list(dataitem)
    except TypeError, value:
        raise StarError( str(dataitem) + ' is wrong type for data value\n' )
    return regval,[None]*len(regval)

def setmaxnamelength(

self, maxlength)

Set the maximum allowable dataname length (-1 for no check)

def setmaxnamelength(self,maxlength):
    """Set the maximum allowable dataname length (-1 for no check)"""
    self.maxnamelength = maxlength
    if maxlength > 0:
        bad_names = [a for a in self.keys() if len(a)>self.maxnamelength]
        if len(bad_names)>0:
            raise StarError, 'Datanames too long: ' + `bad_names`

def unassign_dictionary(

self)

Remove dictionary-dependent behaviour

def unassign_dictionary(self):
    """Remove dictionary-dependent behaviour"""
    self.dictionary = None

def update(

self, adict)

def update(self,adict):
    for key in adict.keys():
        self.AddItem(key,adict[key])

def values(

self)

def values(self):
    return [self[a] for a in self.keys()]

class StarDict

class StarDict(dict):
    pass

Ancestors (in MRO)

  • StarDict
  • __builtin__.dict
  • __builtin__.object

class StarError

class StarError(Exception):
    def __init__(self,value):
        self.value = value
    def __str__(self):
        return '\nStar Format error: '+ self.value 

Ancestors (in MRO)

  • StarError
  • exceptions.Exception
  • exceptions.BaseException
  • __builtin__.object

Class variables

var args

var message

Instance variables

var value

Methods

def __init__(

self, value)

def __init__(self,value):
    self.value = value

class StarFile

class StarFile(BlockCollection):
    def __init__(self,datasource=None,maxinlength=-1,maxoutlength=0,
                scoping='instance',grammar='1.1',scantype='standard',
                **kwargs):
        super(StarFile,self).__init__(datasource=datasource,**kwargs)
        self.my_uri = getattr(datasource,'my_uri','')
        if maxoutlength == 0:
            self.maxoutlength = 2048 
        else:
            self.maxoutlength = maxoutlength
        self.scoping = scoping
        if isinstance(datasource,basestring) or hasattr(datasource,"read"):
            ReadStar(datasource,prepared=self,grammar=grammar,scantype=scantype,
            maxlength = maxinlength)
        self.header_comment = \
"""#\\#STAR
##########################################################################
#               STAR Format file 
#               Produced by PySTARRW module
# 
#  This is a STAR file.  STAR is a superset of the CIF file type.  For
#  more information, please refer to International Tables for Crystallography,
#  Volume G, Chapter 2.1
#
##########################################################################
"""
    def set_uri(self,my_uri): self.my_uri = my_uri

Ancestors (in MRO)

Instance variables

var PC

Inheritance: BlockCollection.PC

var blocktype

Inheritance: BlockCollection.blocktype

var characterset

Inheritance: BlockCollection.characterset

var child_table

Inheritance: BlockCollection.child_table

var dictionary

Inheritance: BlockCollection.dictionary

var header_comment

Inheritance: BlockCollection.header_comment

var lower_keys

Inheritance: BlockCollection.lower_keys

var my_uri

var parent_id

Inheritance: BlockCollection.parent_id

var renamed

Inheritance: BlockCollection.renamed

var scoping

Inheritance: BlockCollection.scoping

var standard

Inheritance: BlockCollection.standard

var visible_keys

Inheritance: BlockCollection.visible_keys

Methods

def __init__(

self, datasource=None, maxinlength=-1, maxoutlength=0, scoping='instance', grammar='1.1', scantype='standard', **kwargs)

Inheritance: BlockCollection.__init__

def __init__(self,datasource=None,maxinlength=-1,maxoutlength=0,
            scoping='instance',grammar='1.1',scantype='standard',
            **kwargs):
    super(StarFile,self).__init__(datasource=datasource,**kwargs)
    self.my_uri = getattr(datasource,'my_uri','')
    if maxoutlength == 0:
        self.maxoutlength = 2048 
    else:
        self.maxoutlength = maxoutlength
    self.scoping = scoping
    if isinstance(datasource,basestring) or hasattr(datasource,"read"):
        ReadStar(datasource,prepared=self,grammar=grammar,scantype=scantype,
        maxlength = maxinlength)
    self.header_comment = \
\\#STAR
######################################################################
            STAR Format file 
            Produced by PySTARRW module
his is a STAR file.  STAR is a superset of the CIF file type.  For
ore information, please refer to International Tables for Crystallography,
olume G, Chapter 2.1
######################################################################

def NewBlock(

self, blockname, blockcontents=None, fix=True, parent=None)

Inheritance: BlockCollection.NewBlock

def NewBlock(self,blockname,blockcontents=None,fix=True,parent=None):
    if blockcontents is None:
        blockcontents = StarBlock()
    if self.standard == "CIF":
        blockcontents.setmaxnamelength(75)
    if len(blockname)>75:
             raise StarError , 'Blockname %s is longer than 75 characters' % blockname
    if fix:
        newblockname = re.sub('[  \t]','_',blockname)
    else: newblockname = blockname
    new_lowerbn = newblockname.lower()
    if new_lowerbn in self.lower_keys:
        if self.standard is not None:    #already there
           toplevelnames = [a[0] for a in self.child_table.items() if a[1].parent==None]
           if parent is None and new_lowerbn not in toplevelnames:  #can give a new key to this one
              while new_lowerbn in self.lower_keys: new_lowerbn = new_lowerbn + '+'
           elif parent is not None and new_lowerbn in toplevelnames: #can fix a different one
              replace_name = new_lowerbn            
              while replace_name in self.lower_keys: replace_name = replace_name + '+'
              self._rekey(new_lowerbn,replace_name)
              # now continue on to add in the new block
              if parent.lower() == new_lowerbn:        #the new block's requested parent just got renamed!!
                  parent = replace_name
           else:
              raise StarError( "Attempt to replace existing block " + blockname)
        else:
           del self[new_lowerbn]
    self.dictionary.update({new_lowerbn:blockcontents})
    self.lower_keys.add(new_lowerbn)
    if parent is None:
       self.child_table[new_lowerbn]=self.PC(newblockname,None)
       self.visible_keys.append(new_lowerbn)
    else:
       if parent.lower() in self.lower_keys:
          if self.scoping == 'instance':
             self.child_table[new_lowerbn]=self.PC(newblockname,parent.lower())
          else:
             self.child_table[new_lowerbn]=self.PC(newblockname,parent.lower())
             self.visible_keys.append(new_lowerbn)
       else:
           print 'Warning:Parent block %s does not exist for child %s' % (parent,newblockname)  
    return new_lowerbn  #in case calling routine wants to know

def WriteOut(

self, comment='', wraplength=80, maxoutlength=2048)

Inheritance: BlockCollection.WriteOut

def WriteOut(self,comment='',wraplength=80,maxoutlength=2048):
    import cStringIO
    if not comment:
        comment = self.header_comment
    outstring = cStringIO.StringIO()
    outstring.write(comment)
    # loop over top-level
    top_block_names = [(a[0],a[1].block_id) for a in self.child_table.items() if a[1].parent is None]
    for blockref,blockname in top_block_names:
        outstring.write('\n' + 'data_' +blockname+'\n')
        child_names = [(a[0],a[1].block_id) for a in self.child_table.items() if a[1].parent==blockref]
        if self.standard == 'Dic':              #put contents before save frames
          self[blockref].SetOutputLength(wraplength,maxoutlength)
          outstring.write(str(self[blockref]))
        for child_ref,child_name in child_names:
            outstring.write('\n' + 'save_' + child_name + '\n')
            self.block_to_string(child_ref,child_name,outstring,4)    
            outstring.write('\n' + 'save_'+ '\n')   
        if self.standard != 'Dic':              #put contents after save frames
            self[blockref].SetOutputLength(wraplength,maxoutlength)
            outstring.write(str(self[blockref]))
    returnstring =  outstring.getvalue()
    outstring.close()
    return returnstring

def block_to_string(

self, block_ref, block_id, outstring, indentlevel=0)

Inheritance: BlockCollection.block_to_string

Output a complete datablock indexed by [[block_ref]] and named [[block_id]], including children

def block_to_string(self,block_ref,block_id,outstring,indentlevel=0):
    """Output a complete datablock indexed by [[block_ref]] and named [[block_id]], including children"""
    child_names = [(a[0],a[1].block_id) for a in self.child_table.items() if a[1].parent==block_ref]
    if self.standard == 'Dic':
        outstring.write(str(self[block_ref]))
    for child_ref,child_name in child_names:
        outstring.write('\n' + 'save_' + child_name + '\n')
        self.block_to_string(child_ref,child_name,outstring,indentlevel)
        outstring.write('\n' + '  '*indentlevel + 'save_' + '\n')
    if self.standard != 'Dic':
        outstring.write(str(self[block_ref]))

def checknamelengths(

self, target_block, maxlength=-1)

Inheritance: BlockCollection.checknamelengths

def checknamelengths(self,target_block,maxlength=-1):
    if maxlength < 0:
        return
    else:
        toolong = filter(lambda a:len(a)>maxlength, target_block.keys())
    outstring = ""
    for it in toolong: outstring += "\n" + it
    if toolong:
       raise StarError( 'Following data names too long:' + outstring)

def clear(

self)

Inheritance: BlockCollection.clear

def clear(self):
    self.dictionary.clear()
    self.lower_keys = set()
    self.child_table = {}
    self.visible_keys = []

def copy(

self)

Inheritance: BlockCollection.copy

def copy(self):   
    newcopy = self.dictionary.copy()  #all blocks
    newcopy = BlockCollection('',newcopy,parent_id=self.parent_id)
    newcopy.child_table = self.child_table.copy()
    newcopy.lower_keys = self.lower_keys
    newcopy.characterset = self.characterset
    newcopy.scoping = self.scoping  #this sets visible keys
    return newcopy

def first_block(

self)

Inheritance: BlockCollection.first_block

Return the 'first' block. This is not necessarily the first block in the file.

def first_block(self):
    """Return the 'first' block.  This is not necessarily the first block in the file."""
    if self.keys():
        return self[self.keys()[0]]

def get(

self, key, default=None)

Inheritance: BlockCollection.get

def get(self,key,default=None):
    if self.has_key(key):     # take account of case
        return self.__getitem__(key)
    else:
        return default

def get_all(

self, item_name)

Inheritance: BlockCollection.get_all

def get_all(self,item_name):
    raw_values = map(lambda a:self[a].get(item_name),self.keys())
    raw_values = filter(lambda a:a != None, raw_values)
    ret_vals = []
    for rv in raw_values:
        if isinstance(rv,list):
            for rvv in rv:
                if rvv not in ret_vals: ret_vals.append(rvv)
        else:
            if rv not in ret_vals: ret_vals.append(rv)
    return ret_vals

def get_child_list(

self, parentname)

Inheritance: BlockCollection.get_child_list

Get a list of all child categories

def get_child_list(self,parentname):
    """Get a list of all child categories"""
    child_handles = [a[0] for a in self.child_table.items() if self.is_child_of_parent(parentname.lower(),a[0])]
    return child_handles

def get_children(

self, blockname, include_parent=False, scoping='dictionary')

Inheritance: BlockCollection.get_children

Get all children of [[blockname]] as a block collection. If [[include_parent]] is True, the parent block will also be included in the block collection as the root.

def get_children(self,blockname,include_parent=False,scoping='dictionary'):
    """Get all children of [[blockname]] as a block collection. If [[include_parent]] is
    True, the parent block will also be included in the block collection as the root."""
    newbc = BlockCollection()
    block_lower = blockname.lower()
    proto_child_table = [a for a in self.child_table.items() if self.is_child_of_parent(block_lower,a[1].block_id)]
    newbc.child_table = dict(proto_child_table)
    if not include_parent:
       newbc.child_table.update(dict([(a[0],self.PC(a[1].block_id,None)) for a in proto_child_table if a[1].parent == block_lower]))
    newbc.lower_keys = set([a[0] for a in proto_child_table])
    newbc.dictionary = dict((a[0],self.dictionary[a[0]]) for a in proto_child_table)
    if include_parent:
        newbc.child_table.update({block_lower:self.PC(self.child_table[block_lower].block_id,None)})
        newbc.lower_keys.add(block_lower)
        newbc.dictionary.update({block_lower:self.dictionary[block_lower]})
    newbc.scoping = scoping
    return newbc

def get_immediate_children(

self, parentname)

Inheritance: BlockCollection.get_immediate_children

Get the next level of children of the given block as a list, without nested levels

def get_immediate_children(self,parentname):
    """Get the next level of children of the given block as a list, without nested levels"""
    child_handles = [a for a in self.child_table.items() if a[1].parent == parentname.lower()]
    return child_handles

def get_parent(

self, blockname)

Inheritance: BlockCollection.get_parent

Return the name of the block enclosing [[blockname]] in canonical form (lower case)

def get_parent(self,blockname):
    """Return the name of the block enclosing [[blockname]] in canonical form (lower case)"""
    possibles = (a for a in self.child_table.items() if a[0] == blockname.lower())
    try:
        first = possibles.next()   #get first one
    except:
        raise StarError('no parent for %s' % blockname)
    try:
       second = possibles.next()
    except StopIteration:
       return first[1].parent
    raise StarError('More than one parent for %s' % blockname)

def get_roots(

self)

Inheritance: BlockCollection.get_roots

Get the top-level blocks

def get_roots(self):
    """Get the top-level blocks"""
    return [a for a in self.child_table.items() if a[1].parent==None]

def has_key(

self, key)

Inheritance: BlockCollection.has_key

def has_key(self,key):
    if not isinstance(key,basestring): return 0
    if key.lower() in self.visible_keys:
        return 1
    return 0

def is_child_of_parent(

self, parentname, blockname)

Inheritance: BlockCollection.is_child_of_parent

Recursively search for children of blockname, case is important for now

def is_child_of_parent(self,parentname,blockname):
    """Recursively search for children of blockname, case is important for now"""
    checkname = parentname.lower()
    more_children = [a[0] for a in self.child_table.items() if a[1].parent == checkname]
    if blockname.lower() in more_children:
       return True
    else:
       for one_child in more_children:
           if self.is_child_of_parent(one_child,blockname): return True
    return False

def items(

self)

Inheritance: BlockCollection.items

def items(self):
    return [(a,self[a]) for a in self.keys()]

def keys(

self)

Inheritance: BlockCollection.keys

def keys(self):
    return self.visible_keys

def lock(

self)

Inheritance: BlockCollection.lock

Disallow overwriting for all blocks in this collection

def lock(self):
    """Disallow overwriting for all blocks in this collection"""
    for a in self.lower_keys:
        self[a].overwrite = False

def merge(

self, new_bc, mode=None, parent=None, single_block=[], idblock='', match_att=[], match_function=None)

Inheritance: BlockCollection.merge

def merge(self,new_bc,mode=None,parent=None,single_block=[],
               idblock="",match_att=[],match_function=None):
    if mode is None:
        if self.standard is None:
           mode = 'replace'
        else:
           mode = 'strict'
    if single_block:
        self[single_block[0]].merge(new_bc[single_block[1]],mode,
                                               match_att=match_att,
                                               match_function=match_function)
        return None
    base_keys = [a[1].block_id for a in self.child_table.items()]
    block_to_item = base_keys   #default
    new_keys = [a[1].block_id for a in new_bc.child_table.items()]    #get list of incoming blocks
    if match_att:
        #make a blockname -> item name map
        if match_function:
            block_to_item = map(lambda a:match_function(self[a]),self.keys())
        else:
            block_to_item = map(lambda a:self[a].get(match_att[0],None),self.keys())
        #print `block_to_item`
    for key in new_keys:        #run over incoming blocknames
        if key == idblock: continue    #skip dictionary id
        basekey = key           #default value
        if len(match_att)>0:
           attval = new_bc[key].get(match_att[0],0)  #0 if ignoring matching
        else:
           attval = 0
        for ii in range(len(block_to_item)):  #do this way to get looped names
            thisatt = block_to_item[ii]       #keyname in old block
            #print "Looking for %s in %s" % (attval,thisatt)
            if attval == thisatt or \
               (isinstance(thisatt,list) and attval in thisatt):
                  basekey = base_keys.pop(ii)
                  block_to_item.remove(thisatt)
                  break
        if not self.has_key(basekey) or mode=="replace":
            new_parent = new_bc.get_parent(key)
            if parent is not None and new_parent is None:
               new_parent = parent
            self.NewBlock(basekey,new_bc[key],parent=new_parent)   #add the block
        else:
            if mode=="strict":
                raise StarError( "In strict merge mode: block %s in old and block %s in new files" % (basekey,key))
            elif mode=="overlay":
                # print "Merging block %s with %s" % (basekey,key)
                self[basekey].merge(new_bc[key],mode,match_att=match_att)
            else:  
                raise StarError( "Merge called with unknown mode %s" % mode)

def merge_fast(

self, new_bc, parent=None)

Inheritance: BlockCollection.merge_fast

Do a fast merge

def merge_fast(self,new_bc,parent=None):
    """Do a fast merge"""
    if self.standard is None:
        mode = 'replace' 
    else:
        mode = 'strict'
    overlap_flag = not self.lower_keys.isdisjoint(new_bc.lower_keys)
    if overlap_flag and mode != 'replace':
        double_keys = self.lower_keys.intersection(new_bc.lower_keys)
        for dup_key in double_keys:
              our_parent = self.child_table[dup_key].parent
              their_parent = new_bc.child_table[dup_key].parent
              if (our_parent is None and their_parent is not None and parent is None) or\
                  parent is not None:  #rename our block
                start_key = dup_key
                while start_key in self.lower_keys: start_key = start_key+'+'
                self._rekey(dup_key,start_key)
                if parent.lower() == dup_key:  #we just renamed the prospective parent!
                    parent = start_key
              elif our_parent is not None and their_parent is None and parent is None:
                start_key = dup_key
                while start_key in new_bc.lower_keys: start_key = start_key+'+'
                new_bc._rekey(dup_key,start_key)
              else: 
                raise StarError("In strict merge mode:duplicate keys %s" % dup_key)
    self.dictionary.update(new_bc.dictionary) 
    self.lower_keys.update(new_bc.lower_keys)
    self.visible_keys += (list(new_bc.lower_keys))
    self.child_table.update(new_bc.child_table)
    if parent is not None:     #redo the child_table entries
          reparent_list = [(a[0],a[1].block_id) for a in new_bc.child_table.items() if a[1].parent==None]
          reparent_dict = [(a[0],self.PC(a[1],parent.lower())) for a in reparent_list]
          self.child_table.update(dict(reparent_dict))

def rename(

self, oldname, newname)

Inheritance: BlockCollection.rename

Rename datablock from [[oldname]] to [[newname]]. Both key and printed name are changed. No conformance checks are conducted.

def rename(self,oldname,newname):
    """Rename datablock from [[oldname]] to [[newname]]. Both key and printed name are changed.  No
       conformance checks are conducted."""
    realoldname = oldname.lower()
    realnewname = newname.lower()
    if realnewname in self.lower_keys:
        raise StarError,'Cannot change blockname %s to %s as %s already present' % (oldname,newname,newname)
    if realoldname not in self.lower_keys:
        raise KeyError,'Cannot find old block %s' % realoldname
    self._rekey(realoldname,realnewname,block_id=newname)

def set_parent(

self, parentname, childname)

Inheritance: BlockCollection.set_parent

Set the parent block

def set_parent(self,parentname,childname):
    """Set the parent block"""
    # first check that both blocks exist
    if parentname.lower() not in self.lower_keys:
        raise KeyError('Parent block %s does not exist' % parentname)
    if childname.lower() not in self.lower_keys:
        raise KeyError('Child block %s does not exist' % childname)
    old_entry = self.child_table[childname.lower()]
    self.child_table[childname.lower()]=self.PC(old_entry.block_id,
           parentname.lower())
    self.scoping = self.scoping #reset visibility

def set_uri(

self, my_uri)

def set_uri(self,my_uri): self.my_uri = my_uri

def unlock(

self)

Inheritance: BlockCollection.unlock

Allow overwriting of all blocks in this collection

def unlock(self):
    """Allow overwriting of all blocks in this collection"""
    for a in self.lower_keys:
        self[a].overwrite=True

def update(

self, adict)

Inheritance: BlockCollection.update

def update(self,adict):
    for key in adict.keys():
        self[key] = adict[key]

class StarLengthError

class StarLengthError(Exception):
    def __init__(self,value):
        self.value = value
    def __str__(self):
        return '\nStar length error: ' + self.value

Ancestors (in MRO)

  • StarLengthError
  • exceptions.Exception
  • exceptions.BaseException
  • __builtin__.object

Class variables

var args

var message

Instance variables

var value

Methods

def __init__(

self, value)

def __init__(self,value):
    self.value = value

class StarList

class StarList(list):
    pass

Ancestors (in MRO)

  • StarList
  • __builtin__.list
  • __builtin__.object

class StarPacket

class StarPacket(list):
    def merge_packet(self,incoming):
        """Merge contents of incoming packet with this packet"""
        new_attrs = [a for a in dir(incoming) if a[0] == '_' and a[1] != "_"]
        self.append(incoming)
        for na in new_attrs:
            setattr(self,na,getattr(incoming,na))

    def __getattr__(self,att_name):
        """Derive a missing attribute"""
        if att_name.lower() in self.__dict__:
            return getattr(self,att_name.lower())
        if att_name in ('cif_dictionary','fulldata','key'):
            raise AttributeError, 'Programming error: cannot compute value of %s' % att_name
        d = self.cif_dictionary
        c = self.fulldata
        k = self.key
        d.derive_item(att_name,c,store_value=True)
        # 
        # now pick out the new value
        keyval = getattr(self,k)
        full_pack = c.GetKeyedPacket(k,keyval)
        return getattr(full_pack,att_name)

Ancestors (in MRO)

Methods

def merge_packet(

self, incoming)

Merge contents of incoming packet with this packet

def merge_packet(self,incoming):
    """Merge contents of incoming packet with this packet"""
    new_attrs = [a for a in dir(incoming) if a[0] == '_' and a[1] != "_"]
    self.append(incoming)
    for na in new_attrs:
        setattr(self,na,getattr(incoming,na))