In This Package:

SiteConfig.py

Go to the documentation of this file.

# SiteConfig
#
# Contains the site configuration for the current cluster/computer
#
# Created by: dandwyer@caltech.edu 2010/08/09

import ProcessManager.Status as Status

class SiteConfig:
    """ Configuration information for a given computer site """
    def __init__(self):
        """Constructor"""
        import os
        self.nuwaDir=None
        self.cmtConfig=None
        self.outputDir=None
        self.jobLogDir=None
        self.statsOffsetDir=None
        self.figuresOffsetDir=None
        self.xmlOffsetDir=None
        self.statsRootDir=None
        self.figuresRootDir=None
        self.xmlRootDir=None
        self.runsXmlDir=None
        self.batchExec=None
        self.batchWrapper="$PROCESSMANAGERROOT/scripts/batchNuWa.sh"
        self.batchLogDir=self.jobLogDir
        self.batchLogDir=None
        self.stateRootDir=None
        return

    def jobId(self, jobName=None, runNumber=None, seqNumber=None):
        """ Return a string to identify the job """
        jobId = ""
        if jobName:
            jobId += jobName
        else:
            jobId += "noname"
        if runNumber:
            jobId += "_run%07d" % runNumber
        if seqNumber:
            jobId += "_seq%04d" % seqNumber
        return jobId
        
    def batchCommand(self, batchArgs, jobName, runNumber, seqNumber):
        """Construct a batch command"""
        print "SiteConfig: batchCommand is undefined for this site"
        return ""

    def runSepDir(self, runNumber):
        """Make an appropriate structure for run subdirectories"""
        # group runs into subdirectories of 1000/100
        runK = runNumber / 1000
        runH = runNumber / 100
        return "runs_%04d000/runs_%05d00" % (runK, runH)

    def runSubDir(self, runNumber):
        """Return run subpath + directory"""
        return self.runSepDir(runNumber)+"/run_%07d" % runNumber

    def statsRelativeDir(self, runNumber):
        """Construct the relative path of the root files from the index"""
        return self.runSubDir(runNumber)+"/root"

    def statsDirectory(self, runNumber):
        """Construct the absolute path of the root files from the index"""
        return self.statsRootDir+"/"+self.statsRelativeDir(runNumber)

    def statsSeqFile(self, jobName, runNumber, seqNumber):
        """Construct the stats ROOT file path and name"""
        filename = "%s.root" % self.jobId(jobName, runNumber, seqNumber)
        return self.statsDirectory(runNumber)+"/"+filename

    def statsInstallPath(self, jobName, runNumber):
        """Construct the relative summed stats ROOT file path and name for index lookup"""
        filename = "%s_total.root" % self.jobId(jobName, runNumber)
        return self.statsOffsetDir+'/'+self.statsRelativeDir(runNumber)+"/"+filename

    def statsSumFile(self, jobName, runNumber):
        """Construct the summed stats ROOT file path and name"""
        filename = "%s_total.root" % self.jobId(jobName, runNumber)
        return self.statsDirectory(runNumber)+"/"+filename

    def statsLockFile(self, jobName, runNumber):
        """Construct the stats lock file path and name"""
        filename = "%s_total.lock" % self.jobId(jobName, runNumber)
        return self.statsDirectory(runNumber)+"/"+filename

    def runLockFile(self, runNumber):
        """Construct the run summary lock file path and name"""
        filename = "run_%07d.lock" % runNumber
        return self.xmlRootDir+"/"+self.runSubDir(runNumber)+"/"+filename

    def figuresDirectory(self, runNumber):
        """Construct the figures directory for the given run"""
        return self.figuresRootDir+"/"+self.runSubDir(runNumber)

    def figuresPathReplace(self, runNumber):
        """Construct the figures directory for the given run"""
        runPath = "run_%07d" % runNumber
        return runPath+","+self.runSubDir(runNumber)

    def xmlRelativeDir(self, runNumber):
        """Construct the xml directory for the given run"""
        return self.xmlOffsetDir+"/"+self.runSubDir(runNumber)

    def xmlDirectory(self, runNumber):
        """Construct the xml directory for the given run"""
        return self.outputDir+'/'+self.xmlRelativeDir(runNumber)

    def xmlIndexFile(self, runNumber):
        """Construct the run index file path and name"""
        filename = "run_%07d.xml" % runNumber
        return self.xmlDirectory(runNumber)+"/"+filename

    def runIndexDir(self):
        """Construct the index of all runs file path"""
        return self.runsXmlDir

    def runIndexFile(self):
        """Construct the index of all runs file path and name"""
        return self.runIndexDir() + "/runs.xml"

    def runIndexLockFile(self):
        """Construct the index of all runs lock file path and name"""
        return self.runsXmlDir + "/runs.lock"

    def logDirectory(self, runNumber):
        """Construct the path for the nuwa log output directory"""
        return self.jobLogDir+"/"+self.runSepDir(runNumber)

    def logOutFile(self, jobName, runNumber, seqNumber=None):
        """Construct the path for the nuwa log output file"""
        import os
        filename = "%s_%d.out" % (self.jobId(jobName, runNumber, seqNumber),
                                  os.getpid() )
        return self.logDirectory(runNumber)+"/"+filename

    def logErrFile(self, jobName, runNumber, seqNumber=None):
        """Construct the path for the log error file"""
        import os
        filename = "%s_%d.err" % (self.jobId(jobName, runNumber, seqNumber),
                                  os.getpid() )
        return self.logDirectory(runNumber)+"/"+filename

    def batchLogDirectory(self, runNumber):
        """Construct the directory for batch log files"""
        return self.batchLogDir+"/"+self.runSepDir(runNumber)

    def batchLogOutFile(self, jobName, runNumber, seqNumber=None):
        """Construct the path for the batch log output file"""
        import os
        jobId = self.jobId(jobName, runNumber, seqNumber)
        filename = "batch_%s_%d.out" % (self.jobId(jobName, runNumber,
                                                   seqNumber),
                                        os.getpid() )
        return self.batchLogDirectory(runNumber)+"/"+filename

    def batchLogErrFile(self, jobName, runNumber, seqNumber=None):
        """Construct the path for the batch log error file"""
        import os
        filename = "batch_%s_%d.err" % (self.jobId(jobName, runNumber,
                                                   seqNumber),
                                        os.getpid() )
        return self.batchLogDirectory(runNumber)+"/"+filename

    def stateFile(self, jobType, jobName, runNumber, seqNumber):
        """Construct the path and filename for logging state of this job"""
        filename = "%s_%s.state" % (self.jobId(jobName, runNumber,seqNumber),
                                    jobType)
        return self.stateRootDir+"/"+self.runSubDir(runNumber)+"/"+filename
    
class SiteConfigLocal(SiteConfig):
    """ Configuration information for a local install """
    def __init__(self):
        """Constructor"""
        SiteConfig.__init__(self)
        self.nuwaDir="$SITEROOT"
        self.cmtConfig="$CMTCONFIG"
        self.outputDir="."
        self.jobLogDir=self.outputDir+"/logs"
        self.statsOffsetDir="diagnostics"
        self.figuresOffsetDir="diagnostics"
        self.xmlOffsetDir="diagnostics"
        self.statsRootDir=self.outputDir+'/'+self.statsOffsetDir
        self.figuresRootDir=self.outputDir+'/'+self.figuresOffsetDir
        self.xmlRootDir=self.outputDir+'/'+self.xmlOffsetDir
        self.runsXmlDir=self.outputDir
        self.batchExec=None
        self.batchLogDir=self.jobLogDir
        self.stateRootDir=self.outputDir+"/state"
        
    def batchCommand(self, batchArgs, jobName, runNumber, seqNumber):
        """Construct a batch command"""
        import os
        pid = os.getpid()
        batchCommand = "%(wrapper)s %(nuwaDir)s %(cmtConfig)s " % {
            'wrapper':self.batchWrapper,
            'nuwaDir':self.nuwaDir,
            'cmtConfig':self.cmtConfig
            }
        batchCommand += "'runProcess.py %(args)s " % {
            'args': ' '.join(batchArgs) }
        batchCommand += "> %(outlog)s 2> %(errlog)s &'" % {
            'outlog':self.batchLogOutFile(jobName, runNumber, seqNumber),
            'errlog':self.batchLogErrFile(jobName, runNumber, seqNumber) }
        return batchCommand

class SiteConfigPdsf(SiteConfig):
    """ Configuration information for a given site """
    def __init__(self):
        """Constructor"""
        SiteConfig.__init__(self)
        #self.nuwaDir="/common/dayabay/releases/NuWa/trunk-opt/NuWa-trunk"
        self.nuwaDir="/eliza7/dayabay/scratch/dandwyer/NuWa-trunk-opt"
        self.cmtConfig="x86_64-slc4-gcc34-opt"
        self.outputDir="/project/projectdirs/dayabay/www/dybprod"
        self.jobLogDir=self.outputDir+"/logs"
        self.statsOffsetDir="diagnostics"
        self.figuresOffsetDir="diagnostics"
        self.xmlOffsetDir="diagnostics"
        self.statsRootDir=self.outputDir+'/'+self.statsOffsetDir
        self.figuresRootDir=self.outputDir+'/'+self.figuresOffsetDir
        self.xmlRootDir=self.outputDir+'/'+self.xmlOffsetDir
        self.runsXmlDir=self.outputDir
        self.batchExec="qsub"
        self.batchLogDir=self.jobLogDir
        self.stateRootDir=self.outputDir+"/state"
        
    def batchCommand(self, batchArgs, jobName, runNumber, seqNumber):
        """Construct a batch command"""
        import os
        pid = os.getpid()
        jobId = "%s_%d" % (jobName[:2],runNumber)
        if seqNumber:
            jobId += "_%d" % seqNumber
        batchCommand = "%(exec)s -o %(outlog)s -e %(errlog)s -N %(jobId)s " % {
            'exec':self.batchExec,
            'outlog':self.batchLogOutFile(jobName, runNumber, seqNumber),
            'errlog':self.batchLogErrFile(jobName, runNumber, seqNumber),
            'jobId':jobId}
        batchCommand += "%(wrapper)s %(nuwaDir)s %(cmtConfig)s " % {
            'wrapper':self.batchWrapper,
            'nuwaDir':self.nuwaDir,
            'cmtConfig':self.cmtConfig
            }
        batchCommand += "'runProcess.py %(args)s'" % {
            'args': ' '.join(batchArgs) }  
        return batchCommand

class SiteConfigPdsfTest(SiteConfig):
    """ Configuration information for a given site """
    def __init__(self):
        """Constructor"""
        SiteConfig.__init__(self)
        self.nuwaDir="$SITEROOT"
        self.cmtConfig="$CMTCONFIG"
        self.outputDir="/eliza7/dayabay/scratch/dandwyer/dybprodTest"
        self.jobLogDir=self.outputDir+"/logs"
        self.statsOffsetDir="diagnostics"
        self.figuresOffsetDir="diagnostics"
        self.xmlOffsetDir="diagnostics"
        self.statsRootDir=self.outputDir+'/'+self.statsOffsetDir
        self.figuresRootDir=self.outputDir+'/'+self.figuresOffsetDir
        self.xmlRootDir=self.outputDir+'/'+self.xmlOffsetDir
        self.runsXmlDir=self.outputDir
        self.batchExec="qsub"
        self.batchLogDir=self.jobLogDir
        self.stateRootDir=self.outputDir+"/state"
        
    def batchCommand(self, batchArgs, jobName, runNumber, seqNumber):
        """Construct a batch command"""
        import os
        pid = os.getpid()
        jobId = "%s_%d" % (jobName[:2],runNumber)
        if seqNumber:
            jobId += "_%d" % seqNumber
        batchCommand = "%(exec)s -o %(outlog)s -e %(errlog)s -N %(jobId)s " % {
            'exec':self.batchExec,
            'outlog':self.batchLogOutFile(jobName, runNumber, seqNumber),
            'errlog':self.batchLogErrFile(jobName, runNumber, seqNumber),
            'jobId':jobId}
        batchCommand += "%(wrapper)s %(nuwaDir)s %(cmtConfig)s " % {
            'wrapper':self.batchWrapper,
            'nuwaDir':self.nuwaDir,
            'cmtConfig':self.cmtConfig
            }
        batchCommand += "'runProcess.py %(args)s'" % {
            'args': ' '.join(batchArgs) }  
        return batchCommand

class SiteConfigPdsfSim(SiteConfig):
    """ Configuration information for a given site """
    def __init__(self):
        """Constructor"""
        SiteConfig.__init__(self)
        #self.nuwaDir="/common/dayabay/releases/NuWa/trunk-opt/NuWa-trunk"
        self.nuwaDir="$SITEROOT"
        self.cmtConfig="$CMTCONFIG"
        self.outputDir="/project/projectdirs/dayabay/www/dybprodSim"
        self.jobLogDir=self.outputDir+"/logs"
        self.statsOffsetDir="diagnostics"
        self.figuresOffsetDir="diagnostics"
        self.xmlOffsetDir="diagnostics"
        self.statsRootDir=self.outputDir+'/'+self.statsOffsetDir
        self.figuresRootDir=self.outputDir+'/'+self.figuresOffsetDir
        self.xmlRootDir=self.outputDir+'/'+self.xmlOffsetDir
        self.runsXmlDir=self.outputDir
        self.batchExec="qsub"
        self.batchLogDir=self.jobLogDir
        self.stateRootDir=self.outputDir+"/state"
        
    def batchCommand(self, batchArgs, jobName, runNumber, seqNumber):
        """Construct a batch command"""
        import os
        pid = os.getpid()
        jobId = "%s_%d" % (jobName[:2],runNumber)
        if seqNumber:
            jobId += "_%d" % seqNumber
        batchCommand = "%(exec)s -o %(outlog)s -e %(errlog)s -N %(jobId)s " % {
            'exec':self.batchExec,
            'outlog':self.batchLogOutFile(jobName, runNumber, seqNumber),
            'errlog':self.batchLogErrFile(jobName, runNumber, seqNumber),
            'jobId':jobId}
        batchCommand += "%(wrapper)s %(nuwaDir)s %(cmtConfig)s " % {
            'wrapper':self.batchWrapper,
            'nuwaDir':self.nuwaDir,
            'cmtConfig':self.cmtConfig
            }
        batchCommand += "'runProcess.py %(args)s'" % {
            'args': ' '.join(batchArgs) }  
        return batchCommand


siteConfigs = {"local":SiteConfigLocal(),
               "pdsf":SiteConfigPdsf(),
               "pdsfSim":SiteConfigPdsfSim(),
               "pdsfTest":SiteConfigPdsfTest()
               }

---