U
    –rf-k  ã                   @   s€  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ e e¡ZdHdd„Zdd„ Zdd	„ Ze	d
d„ ƒZdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ ZdIdd„Zdd„ Zd d!„ Zd"d#„ Zd$d%„ Zd&d'„ Z d(d)„ Z!d*d+„ Z"d,d-„ Z#d.d/„ Z$d0d1„ Z%d2d3„ Z&d4d5„ Z'd6d7„ Z(dJd8d9„Z)d:d;„ Z*d<d=„ Z+d>d?„ Z,d@dA„ Z-dBdC„ Z.dDdE„ Z/dFdG„ Z0dS )Ké    N)Úcontextmanager)Úutilsc              	   C   s„   t  ¡ r |d krtjd n|}|  d¡}|  d¡}|  d¡}t|||ƒ t|||ƒ t||| ƒ t||ƒ t	||ƒ W 5 Q R X d S )NÚLIDÚSRCROOTÚEXEROOTÚCASEROOT)
r   Ú
SharedAreaÚosÚenvironÚ	get_valueÚ_record_git_provenanceÚ_archive_source_modsÚ_archive_build_environmentÚ_archive_build_timesÚ_symlink_current)ÚcaseÚlidÚsrcrootÚexerootÚcaseroot© r   úL/global/u1/q/qinyi/eagles_code/eaglesrrm/cime_config/customize/provenance.pyÚsave_build_provenance   s    




r   c              	   C   sœ  t j |d |¡¡}tjd| d}t|dƒ}| |¡ W 5 Q R X t| ƒ}t j |dd¡}t j |d |¡¡}t j 	|¡r„t  
|¡ t j 	|¡r tj||dd	 t j |d
 |¡¡}	tjd| d}
t|	dƒ}| |
¡ W 5 Q R X t j |d |¡¡}td| |ƒ t j |d |¡¡}td| |ƒ t j |d |¡¡}d}t|| |ƒ t j |d |¡¡}td| |ƒ t j |d¡}t j |d |¡¡}tj||dd	 dS )zdRecords git provenance

    Records git status, diff and logs for main repo and all submodules.
    úGIT_DESCRIBE.{}T©ÚtagÚrepoÚwÚlogsÚHEADzGIT_LOGS_HEAD.{}F©Úpreserve_metazGIT_SUBMODULE_STATUS.{})Ú	recursiver   zGIT_STATUS.{}ÚstatuszGIT_DIFF.{}Údiffz
GIT_LOG.{}z(log --first-parent --pretty=oneline -n 5zGIT_REMOTE.{}z	remote -vÚconfigzGIT_CONFIG.{}N)r	   ÚpathÚjoinÚformatr   Úget_current_commitÚopenÚwriteÚ_find_git_rootÚexistsÚremoveÚ	safe_copyÚget_current_submodule_statusÚ_run_git_cmd_recursively)r   r   r   Zdescribe_provÚdescÚfdÚgitrootZheadfileZheadfile_provZsubmodule_provZsubm_statusZstatus_provZ	diff_provZlog_provÚcmdZremote_provZ
config_srcZconfig_provr   r   r   r   #   s6    
r   c              	   C   s‚   | › d}t  tj |¡| ›d¡ tj |¡r4|S t| ƒ tj t|ƒ¡}W 5 Q R X tj 	|d¡}tj |¡rv|S t
|ƒ}|S )aR  Finds the `.git` directory.

    NOTICE: It's assumed `srcroot` is an absolute path.

    There are three scenarios to locate the correct `.git` directory:

    NOTICE: In the 3rd case git `.git` directory is not actually `.git`.

    1. In a standard git repository it will be located at `{srcroot}/.git`.
    2. In a git worktree the `{srcroot}/.git` is a file containing a path,
       `{gitdir}`, which the `{gitroot}` can be parsed from.
    3. In a git submodule the `{srcroot}/.git` is a file containing a path,
       `{gitdir}`, where the `{gitroot}` is `{gitdir}`.

    To aid in finding the correct `{gitroot}` the file `{gitroot}/config`
    is checked, this file will always be located in the correct directory.

    Args:
        srcroot (str): Path to the source root.

    Returns:
        str: Absolute path to `.git` directory.
    z/.gitz6 is not a git repository, failed to collect provenancer%   )r   Úexpectr	   r&   r-   ÚisdirÚ	_swap_cwdÚabspathÚ_read_gitdirr'   Ú_parse_dot_git_path)r   r4   ÚgitdirÚconfig_pathZparsed_gitrootr   r   r   r,   U   s    

þ
r,   c              	   c   s.   t  ¡ }t  | ¡ z
d V  W 5 t  |¡ X d S )N)r	   ÚgetcwdÚchdir)Znew_cwdZold_cwdr   r   r   r8   ˆ   s
    

r8   c           
   	   C   s‚   t jd | ¡|d\}}}t jd | ¡|d\}}}t|dƒ:}	|	 |dkrP|n|d ¡ |	 |dkrj|n|d ¡ W 5 Q R X dS )	zªRuns a git command recursively

    Runs the git command in srcroot then runs it on each submodule.
    Then output from both commands is written to the output file.
    zgit {}©Úfrom_dirz0git submodule foreach --recursive "git {}; echo"r   r   z

Ú
N)r   Úrun_cmdr(   r*   r+   )
r5   r   ÚoutputZrc1Zoutput1Zerr1Zrc2Zoutput2Zerr2r3   r   r   r   r1   ”   s     ÿr1   c              	   C   sh   t  tj | ¡d| ›d¡ t| ƒ}| ¡ }W 5 Q R X d}t ||¡}t  |dk	d| ›¡ | 	d¡S )zäRead `gitdir` from `.git` file.

    Reads `.git` file in a worktree or submodule and parse `gitdir`.

    Args:
        gitroot (str): Path ending with `.git` file.

    Returns:
        str: Path contained in `.git` file.
    z	Expected z to be a filez^gitdir:\s*(.*)$Nz"Could parse gitdir path from file é   )
r   r6   r	   r&   Úisfiler*   ÚreadlineÚreÚmatchÚgroup)r4   r3   ÚlineZgitdir_patternÚmr   r   r   r:   ¥   s    
r:   c                 C   s0   d}t  || ¡}t |dk	d| ›¡ | d¡S )a  Parse `.git` path.

    Take a path e.g. `/storage/cime/.git/worktrees/cime` and parse `.git`
    directory e.g. `/storage/cime/.git`.

    Args:
        gitdir (str): Path containing `.git` directory.

    Returns:
        str: Path ending with `.git`
    z^(.*/\.git).*NzCould not parse .git from path rE   )rH   rI   r   r6   rJ   )r<   Zdot_git_patternrL   r   r   r   r;   ¾   s    r;   c              	   C   sn   t j |d¡}t j |d | ¡¡}t j |¡r8t  |¡ t j |¡rjt |d¡}|j	|dd W 5 Q R X d S )NÚ
SourceModsúSourceMods.{}.tar.gzúw:gz©Úarcname)
r	   r&   r'   r(   r-   r.   r7   Útarfiler*   Úadd)r   r   r   Z
sourcemodsZsourcemods_provÚtfdr   r   r   r   Ó   s    
r   c                 C   sB   t j |d | ¡¡}t j |¡r*t  |¡ | d¡}| |¡ d S )Nzbuild_environment.{}.txtÚmach_specific)r	   r&   r'   r(   r-   r.   Úget_envÚsave_all_env_info)r   r   r   Zenv_provÚ
env_moduler   r   r   r   Þ   s
    

r   c                 C   sR   t j |d | ¡¡}t j |¡r*t  |¡ d || ¡}t |¡}|rNt||ƒ d S )Nzbuild_times.{}.txtz{}/*bldlog*{}.gz)r	   r&   r'   r(   r-   r.   ÚglobÚ_extract_times)r   r   Úbuild_timesÚglobstrÚmatchesr   r   r   r   ç   s    

r   c              	   C   sø   d}d}| D ]²}t  d |¡¡\}}}|dkrr| ¡ D ]:}| ¡ }|r6| ¡ }	|	d |	d  }
}|d |
|¡7 }q6t  d |¡¡\}}}|dkr| ¡ D ]$}| ¡ }|r˜|t| ¡ d	 ƒ7 }q˜qt|d
ƒ$}| |¡ | d t	|ƒ¡¡ W 5 Q R X d S )NzTarget Build_time
g        zzgrep 'built in' {}r   rE   éþÿÿÿz{} {}
zzgrep -E '^real [0-9.]+$' {}éÿÿÿÿr   zTotal_Elapsed_Time {})
r   rC   r(   Ú
splitlinesÚstripÚsplitÚfloatr*   r+   Ústr)ZzipfilesZtarget_fileÚcontentsZtotal_build_timeÚzipfileÚstatrD   Ú_rK   ÚitemsÚtargetZthe_timer3   r   r   r   rZ   ò   s(    
rZ   c                 C   s€   dD ]v}d  ||| ¡}t |¡}t t|ƒdk d  |¡¡ |r|d }| d  | ¡d¡}tj |¡rnt 	|¡ t 
||¡ qd S )N)ZGIT_DESCRIBEÚGIT_LOGS_HEADÚGIT_SUBMODULE_STATUSÚ
GIT_STATUSÚGIT_DIFFÚGIT_LOGÚ
GIT_CONFIGÚ
GIT_REMOTErM   Zbuild_environmentr[   z	{}/{}.{}*é   z5Multiple matches for glob {} should not have happenedr   ú.{}Ú )r(   rY   r   r6   ÚlenÚreplacer	   r&   r-   r.   Úsymlink)r   r   Úitemr\   r]   Ú	the_matchZgeneric_namer   r   r   r     s    

þ
r   c              
      s
  t  ¡ ø |d krtjd n|}|  d¡}tj |  d¡d¡}tj |¡sTt 	|¡ | 
tj |d |¡¡¡ |  d¡}tj |d¡}|› d|› }tj |¡rªt |¡ t|d	ƒ&‰ |  ‡ fd
d„d ¡ t  ||¡ W 5 Q R X t| ƒ |  d¡rüt| |ƒ W 5 Q R X d S )Nr   rU   r   r   zrun_environment.txt.{}ÚRUNDIRúpreview_run.logÚ.r   c                    s   ˆ   d | ¡¡S )Nú{}
)r+   r(   )Úx©r3   r   r   Ú<lambda>>  ó    z(save_prerun_provenance.<locals>.<lambda>ÚSAVE_TIMING)r   r   r	   r
   rV   r&   r'   r   r7   ÚmakedirsrW   r(   r-   r.   r*   Úpreview_runr/   Ú_cleanup_spio_statsÚ_record_timing)r   r   rX   ZlogdirZrun_dirZbase_preview_runr„   r   r   r   Úsave_prerun_provenance)  s(    


ÿ


r‡   c                 C   sœ   | d kst j | ¡s$td| › dƒ‚t d | ¡¡ t j | dt 	¡ ||¡}t j 
|¡rft|› dƒ‚zt  |¡ W n" tk
r–   t|› dƒ‚Y nX |S )NzSAVE_TIMING_DIR zL is not valid. E3SM requires a valid SAVE_TIMING_DIR to archive timing data.z6Archiving timing data and associated provenance in {}.Úperformance_archivezK already exists. Skipping archive of timing data and associated provenance.zN cannot be created. Skipping archive of timing data and associated provenance.)r	   r&   r7   ÚRuntimeErrorÚloggerÚinfor(   r'   ÚgetpassÚgetuserr-   rƒ   ÚOSError)Ú
timing_dirÚ	base_caser   Úfull_timing_dirr   r   r   Ú_check_timing_dirI  s2    
ÿÿ    ÿÿÿ
r’   c              
   C   sò  | j d|  ¡ d}|  |¡s d S |   d¡}|   d¡}|   d¡}|   d¡}|   d¡}|   d¡}zt|||ƒ}	W n4 tk
r  }
 zt d	|
¡ W Y ¢d S d }
~
X Y nX |   d
¡}|   d¡}t| ƒ}|d k	rÔt||||	ƒ t	j
 |d¡}t	j
 |¡r&t t	j
 |	d |¡¡d¡}|j|dd W 5 Q R X t	j
 |	d |¡¡}t	 |¡ t|||||ƒ t||	|ƒ t||	|ƒ t	j
 t	j
 |d¡¡rˆ|n
t	j
 |¡}tjd|d}tt	j
 |	d |¡¡dƒ}| |¡ W 5 Q R X |d k	rît| |||||	ƒ d S )NÚPROJECT©Úsubgrouprz   r   r   r   ÚCASEÚSAVE_TIMING_DIRz{!s}ÚMACHÚCOMPILERrM   rN   rO   rP   zCaseDocs.{}z.gitTr   r   r   )r   Úget_primary_jobÚis_save_timing_dir_projectr’   r‰   rŠ   ÚwarningÚ_get_batch_job_id_for_syslogÚ_record_queue_infor	   r&   r'   r7   rR   r*   r(   rS   ÚmkdirÚ_copy_caseroot_filesÚ_copy_blddir_filesÚ_copy_rundir_filesr-   Údirnamer   r)   r+   Ú_record_syslog)r   r   ÚprojectÚrundirÚblddirr   r   r   r   r‘   ÚeÚmachÚcompilerÚjob_idZsource_mods_dirrT   Ú	case_docsZ	from_repor2   r3   r   r   r   r†   f  sR    








 ÿ
ÿ
ý
r†   c                 C   sp   | dkrt |||ƒ nV| dkr,t|||ƒ n@| dkrBt|||ƒ n*| dkrXt|||ƒ n| dkrlt|||ƒ d S )NÚtheta)úcori-haswellúcori-knl)ÚanvilÚ	chrysalisÚcompy)ÚfrontierÚcrusherÚsummit)Ú_record_anl_theta_queueÚ_record_nersc_queueÚ_record_anl_queueÚ_record_slurm_queueÚ_record_olcf_queue)r©   r«   r   r‘   r   r   r   rž   ¥  s    rž   c                 C   sR   dd|  dfddfD ]8\}}d||f }t j|||d t  tj ||¡¡ qd S )N©zsinfo -a -lÚsinfolúscontrol show jobid %sZ
sqsf_jobid©zOsqueue -o '%.10i %.15P %.20j %.10u %.7a %.2t %.6D %.8C %.10M %.10l %.20S %.20V'Úsqueuef©zsqueue -t R -o '%.10i %R'Zsqueuesú%s.%s©Ú
arg_stdoutrA   ©r   Úrun_cmd_no_failÚgzip_existing_filer	   r&   r'   ©r«   r   r‘   r5   Úfilenamer   r   r   r·   ³  s    
ø
r·   c                 C   sR   dd|  dfddfD ]8\}}d||f }t j|||d t  tj ||¡¡ qd S )Nr»   r½   Zscontrol_jobidr¾   rÀ   rÁ   rÂ   rÄ   rÇ   r   r   r   r¹   Ä  s    
ù	r¹   c                 C   sT   dd  | ¡dfddfD ]8\}}d||f }tj|||d t tj ||¡¡ qd S )N)zsinfo -lr¼   zsqueue -o '%all' --job {}Zsqueueall_jobid)zMsqueue -o '%.10i %.10P %.15u %.20a %.2t %.6D %.8C %.12M %.12l %.20S %.20V %j'r¿   rÀ   rÁ   rÂ   )r(   r   rÅ   rÆ   r	   r&   r'   rÇ   r   r   r   r¸   Ô  s    ù	r¸   c                 C   sR   dd|  dfddfD ]8\}}d||f }t j|||d t  tj ||¡¡ qd S )N)z¨qstat -l --header JobID:JobName:User:Project:WallTime:QueuedTime:Score:RunTime:TimeRemaining:Nodes:State:Location:Mode:Command:Args:Procs:Queue:StartTime:attrs:GeometryZqstatfzqstat -lf %sZqstatf_jobid)Ú
xtnodestatrÉ   )ÚxtprocadminrÊ   rÁ   rÂ   rÄ   rÇ   r   r   r   r¶   ã  s    
ù	r¶   c                 C   s^   ddd|  dffD ]F\}}|d | }t j|d | |d t  tj ||d | ¡¡ qd S )N)zbjobs -u all >Z
bjobsu_all)z,bjobs -r -u all -o 'jobid slots exec_host' >Zbjobsru_allozbjobs -l -UF %s >ZbjobslUF_jobidú r|   r@   rÄ   )r«   r   r‘   r5   rÈ   Zfull_cmdr   r   r   rº   ò  s    
ýrº   c                 C   sŠ   |   d¡}t tj |d¡¡D ]}t |¡ qtj |d¡}tj |¡rRt |¡ zt 	|¡ W n$ t
k
r„   t d |¡¡ Y nX d S )Nrz   úio_perf_summary*Ú
spio_statszT{} could not be created. Scorpio I/O statistics will be stored in the run directory.)r   rY   r	   r&   r'   r.   r-   ÚshutilÚrmtreerƒ   rŽ   rŠ   rœ   r(   )r   r¦   rx   Úspio_stats_dirr   r   r   r…   ý  s    

ÿÿr…   c                    sR   ddddddddd	d
  | ¡d
  |¡d  | |¡dg}t ‡ fdd„|D ƒ||¡ d S )Nz
CaseDocs/*zrun_script_provenance/*z*.runz.*.runz*.xmlz	user_nl_*z*env_mach_specific*zMacros*zREADME.casez
Depends.{}zDepends.{}.{}zsoftware_environment.txtc                    s   g | ]}t j ˆ |¡‘qS r   ©r	   r&   r'   ©Ú.0r~   ©r   r   r   Ú
<listcomp>!  s     z(_copy_caseroot_files.<locals>.<listcomp>)r(   r   Ú
copy_globs)r©   rª   r   r¬   r   Úglobs_to_copyr   rÔ   r   r      s    
ór    c              	      s6   ddddddddd	g	}t  ‡ fd
d„|D ƒ||¡ d S )Nrk   rm   rn   ro   rq   rp   rl   zbuild_environment.txtzbuild_times.txtc                    s   g | ]}t j ˆ |¡‘qS r   rÑ   rÒ   ©r§   r   r   rÕ   3  s     z&_copy_blddir_files.<locals>.<listcomp>©r   rÖ   )r§   r‘   r   Zblddir_globs_to_copyr   rØ   r   r¡   $  s    ÷  ÿr¡   c                    s&   dg}t  ‡ fdd„|D ƒ||¡ d S )Nr{   c                    s   g | ]}t j ˆ |¡‘qS r   rÑ   rÒ   ©r¦   r   r   rÕ   =  s     z&_copy_rundir_files.<locals>.<listcomp>rÙ   )r¦   r‘   r   Zrundir_globs_to_copyr   rÚ   r   r¢   7  s    ÿ  ÿr¢   c              
   C   sH  t j |d |¡¡}t j |¡r¢ztz8t|dƒ}t| ¡  	¡ ƒ}W 5 Q R X t  
|tj¡ W n6 ttfk
r }	 zt d |	¡¡ W 5 d }	~	X Y nX W 5 t  |¡ X |  d¡}
|
dkrDt j |d |¡¡}t  |¡ t d ||¡¡ tjdj|
||||d	t j |d
¡d}tt j |d |¡¡dƒ}| d |¡¡ W 5 Q R X d S )Núsyslog_jobid.{}ÚrúFailed to kill syslog: {}ZSYSLOG_Nr   zcheckpoints.{}z{}/e3sm.log.{}za./mach_syslog {si} {jobid} {lid} {rundir} {rundir}/timing/checkpoints {ac} >& /dev/null & echo $!)ÚsiÚjobidr   r¦   ZacÚToolsr@   r   r}   )r	   r&   r'   r(   r-   r.   r*   ÚintÚreadra   ÚkillÚsignalÚSIGTERMÚ
ValueErrorrŽ   rŠ   rœ   r   rŸ   r   ÚtouchrÅ   r+   )r   r   r«   r   r¦   r‘   Úsyslog_jobid_pathr3   Úsyslog_jobidr¨   Zsample_intervalZarchive_checkpointsr   r   r   r¤   A  s:    &

 ÿ
ûø
r¤   c              
   C   sØ  t  ¡ Ä |d krtjd n|}|  d¡rÊ|  d¡}|  d¡}t||ƒ}t||ƒ t||ƒ t||ƒ t  	tj
 |dd| ¡¡ d| }t  tj
 |d|¡¡ | jd|  ¡ d	}|  |¡sÌW 5 Q R £ d S |  d
¡}|d ksêtj
 |¡søW 5 Q R £ d S |  d¡}|  d¡}	tj
 |dt ¡ |	|¡}
tj
 |
¡s@W 5 Q R £ d S t| ƒ}|d k	r\t||ƒ t jd| |
dd t| ||||||
|ƒ t |
¡D ]8\}}}|D ]&}| d¡sžt  	tj
 ||¡¡ qžqW 5 Q R X d S )Nr   r‚   r   rz   Útimingze3sm_timing_stats.%sztiming.%s.savedr“   r”   r—   r˜   r–   rˆ   ú	%s.tar.gzFr    ú.gz)r   r   r	   r
   r   Ú_archive_timingsÚ_archive_atm_costsÚ_archive_memory_profileÚ_archive_spio_statsrÆ   r&   r'   rç   rš   r›   r7   rŒ   r   r   Ú_kill_mach_syslogr/   Ú_copy_performance_archive_filesÚwalkÚendswith)r   r   r   r¦   Úrundir_timing_dirÚtiming_saved_filer¥   r   r©   r   r‘   r«   Úrootrh   ÚfilesrÈ   r   r   r   Úsave_postrun_provenanced  sj    





ÿ



    ÿ

  ÿørù   c              	   C   sh   t j |d|  ¡}t t j |d¡|¡ t d| d¡}|j|t j |¡d W 5 Q R X t 	|¡ |S )Nztiming.rê   rë   rO   rP   )
r	   r&   r'   rÎ   ÚmoverR   r*   rS   ÚbasenamerÏ   )r   r¦   rõ   rT   r   r   r   rí   ©  s     
rí   c                 C   sH   t j |d¡}t j |¡rDt j |d | ¡¡}t ||¡ t |¡ d S )Nzatm_chunk_costs.txtzatm_chunk_costs.{})	r	   r&   r'   r-   r(   rÎ   rú   r   rÆ   )r   r¦   Zatm_chunk_costs_src_pathZatm_chunk_costs_dst_pathr   r   r   rî   µ  s     ÿrî   c                 C   s`   d}t   tj ||¡¡D ]B}tj tj |¡tj |¡d  | ¡¡}t ||¡ t	 
|¡ qd S )Nzmemory.[0-4].*.logrs   )rY   r	   r&   r'   r£   rû   r(   rÎ   rú   r   rÆ   )r   r¦   Úglob_to_copyrx   Zmprof_dst_pathr   r   r   rï   ¿  s    
 ÿrï   c              	   C   s¨   t j |d¡}t j |¡s$t  |¡ t t j |d¡¡D ]}t ||¡ q8t j |d|  ¡}t 	||¡ t
 d| d¡}|j|t j |¡d W 5 Q R X t |¡ d S )NrÍ   rÌ   zspio_stats.rë   rO   rP   )r	   r&   r'   r-   rŸ   rY   r   r/   rÎ   ÚcopytreerR   r*   rS   rû   rÏ   )r   r¦   rÐ   rx   Zspio_stats_job_dirrT   r   r   r   rð   Ê  s    
 rð   c              
   C   s¦   t j |d | ¡¡}t j |¡r¢ztz8t|dƒ}t| ¡  	¡ ƒ}W 5 Q R X t  
|tj¡ W n6 ttfk
r } zt d |¡¡ W 5 d }~X Y nX W 5 t  |¡ X d S )NrÛ   rÜ   rÝ   )r	   r&   r'   r(   r-   r.   r*   rá   râ   ra   rã   rä   rå   ræ   rŽ   rŠ   rœ   )r«   r¦   rè   r3   ré   r¨   r   r   r   rñ   Û  s    &rñ   c                 C   sÒ  g }|d k	r†|dkr.|  d|  d¡|f ¡ nX|dkrb|  d| ¡ |  d| ¡ |  d| ¡ n$|dkr†|  d	| ¡ |  d
| ¡ |  d |¡¡ |  tj |d |¡¡¡ |  tj |d |¡¡¡ |  tj |d |¡¡¡ |  tj |d |¡¡¡ |  d |¡¡ |  d¡ |  tj |d |¡¡¡ |  tj |d¡¡ |D ]‚}	t tj ||	¡¡D ]f}
tj |
¡}||krb||kr¶| d¡s¶t	j
|
tj |d ||¡¡dd nt	j
|
|dd qbqJd S )N)r°   r±   r²   r®   r¯   z	run*%s*%sr–   r­   z%s*errorz	%s*outputz%s*cobaltlogrµ   ze3sm.stderr.%sze3sm.stdout.%szlogs/run_environment.txt.{}ze3sm.log.{}.gzzcpl.log.{}.gzzatm_chunk_costs.{}.gzzmemory.[0-4].*.log.{}.gzztiming/*.{}*Ú
CaseStatuszspio_stats.{}.tar.gzz	replay.shrì   z{}.{}Fr    )Úappendr   r(   r	   r&   r'   rY   rû   rô   r   r/   )r   r   r«   r©   r¦   r   r‘   rö   r×   rü   rx   rû   r   r   r   rò   è  s>    

ýrò   c                 C   sd   |   d¡}z@|dkr tjd W S |dkr4tjd W S |dkrHtjd W S W n tk
r^   Y nX dS )	z4
    mach_syslog only works on certain machines
    r˜   )
r°   r±   r²   r®   r¯   zpm-cpuzpm-gpuZalvarezr³   r´   ÚSLURM_JOB_ID)r­   ÚCOBALT_JOBID)rµ   Ú	LSB_JOBIDN)r   r	   r
   ÚKeyError)r   r©   r   r   r   r     s    
r   )N)N)N)1r	   rR   rY   rH   ÚloggingrŒ   rä   rÎ   Ú
contextlibr   ÚCIMEr   Ú	getLoggerÚ__name__rŠ   r   r   r,   r8   r1   r:   r;   r   r   r   rZ   r   r‡   r’   r†   rž   r·   r¹   r¸   r¶   rº   r…   r    r¡   r¢   r¤   rù   rí   rî   rï   rð   rñ   rò   r   r   r   r   r   Ú<module>   sZ   

23
	
 ?
#
E
'