o
    –rf-k  ã                   @   s€  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ e e¡ZdHdd„Zdd„ Zdd	„ Ze	d
d„ ƒZdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ ZdHdd„Zdd„ Zd d!„ Zd"d#„ Zd$d%„ Zd&d'„ Z d(d)„ Z!d*d+„ Z"d,d-„ Z#d.d/„ Z$d0d1„ Z%d2d3„ Z&d4d5„ Z'd6d7„ Z(dHd8d9„Z)d:d;„ Z*d<d=„ Z+d>d?„ Z,d@dA„ Z-dBdC„ Z.dDdE„ Z/dFdG„ Z0dS )Ié    N)Úcontextmanager)Úutilsc                 C   sš   t  ¡ ? |d u rtjd n|}|  d¡}|  d¡}|  d¡}t|||ƒ t|||ƒ t||| ƒ t||ƒ t	||ƒ W d   ƒ d S 1 sFw   Y  d S )NÚLIDÚSRCROOTÚEXEROOTÚCASEROOT)
r   Ú
SharedAreaÚosÚenvironÚ	get_valueÚ_record_git_provenanceÚ_archive_source_modsÚ_archive_build_environmentÚ_archive_build_timesÚ_symlink_current)ÚcaseÚlidÚsrcrootÚexerootÚcaseroot© r   úL/global/u1/q/qinyi/eagles_code/eaglesrrm/cime_config/customize/provenance.pyÚsave_build_provenance   s   




"ñr   c                 C   sÄ  t j |d |¡¡}tjd| d}t|dƒ}| |¡ W d  ƒ n1 s&w   Y  t| ƒ}t j |dd¡}t j |d |¡¡}t j 	|¡rLt  
|¡ t j 	|¡rZtj||d	d
 t j |d |¡¡}	tjd| d}
t|	dƒ}| |
¡ W d  ƒ n1 s€w   Y  t j |d |¡¡}td| |ƒ t j |d |¡¡}td| |ƒ t j |d |¡¡}d}t|| |ƒ t j |d |¡¡}td| |ƒ t j |d¡}t j |d |¡¡}tj||d	d
 dS )zdRecords git provenance

    Records git status, diff and logs for main repo and all submodules.
    úGIT_DESCRIBE.{}T©ÚtagÚrepoÚwNÚlogsÚHEADzGIT_LOGS_HEAD.{}F©Úpreserve_metazGIT_SUBMODULE_STATUS.{})Ú	recursiver   zGIT_STATUS.{}ÚstatuszGIT_DIFF.{}Údiffz
GIT_LOG.{}z(log --first-parent --pretty=oneline -n 5zGIT_REMOTE.{}z	remote -vÚconfigzGIT_CONFIG.{})r	   ÚpathÚjoinÚformatr   Úget_current_commitÚopenÚwriteÚ_find_git_rootÚexistsÚremoveÚ	safe_copyÚget_current_submodule_statusÚ_run_git_cmd_recursively)r   r   r   Zdescribe_provÚdescÚfdÚgitrootZheadfileZheadfile_provZsubmodule_provZsubm_statusZstatus_provZ	diff_provZlog_provÚcmdZremote_provZ
config_srcZconfig_provr   r   r   r   #   s:   ÿ
ÿr   c                 C   s–   | › d}t  tj |¡| ›d¡ tj |¡r|S t| ƒ tj t|ƒ¡}W d  ƒ n1 s1w   Y  tj 	|d¡}tj |¡rE|S t
|ƒ}|S )aR  Finds the `.git` directory.

    NOTICE: It's assumed `srcroot` is an absolute path.

    There are three scenarios to locate the correct `.git` directory:

    NOTICE: In the 3rd case git `.git` directory is not actually `.git`.

    1. In a standard git repository it will be located at `{srcroot}/.git`.
    2. In a git worktree the `{srcroot}/.git` is a file containing a path,
       `{gitdir}`, which the `{gitroot}` can be parsed from.
    3. In a git submodule the `{srcroot}/.git` is a file containing a path,
       `{gitdir}`, where the `{gitroot}` is `{gitdir}`.

    To aid in finding the correct `{gitroot}` the file `{gitroot}/config`
    is checked, this file will always be located in the correct directory.

    Args:
        srcroot (str): Path to the source root.

    Returns:
        str: Absolute path to `.git` directory.
    z/.gitz6 is not a git repository, failed to collect provenanceNr%   )r   Úexpectr	   r&   r-   ÚisdirÚ	_swap_cwdÚabspathÚ_read_gitdirr'   Ú_parse_dot_git_path)r   r4   ÚgitdirÚconfig_pathZparsed_gitrootr   r   r   r,   U   s   

þ
ÿr,   c              	   c   s8    t  ¡ }t  | ¡ zd V  W t  |¡ d S t  |¡ w ©N)r	   ÚgetcwdÚchdir)Znew_cwdZold_cwdr   r   r   r8   ˆ   s   €
r8   c           
      C   s˜   t jd | ¡|d\}}}t jd | ¡|d\}}}t|dƒ#}	|	 |dkr(|n|d ¡ |	 |dkr5|n|d ¡ W d  ƒ dS 1 sEw   Y  dS )	zªRuns a git command recursively

    Runs the git command in srcroot then runs it on each submodule.
    Then output from both commands is written to the output file.
    zgit {}©Úfrom_dirz0git submodule foreach --recursive "git {}; echo"r   r   z

Ú
N)r   Úrun_cmdr(   r*   r+   )
r5   r   ÚoutputZrc1Zoutput1Zerr1Zrc2Zoutput2Zerr2r3   r   r   r   r1   ”   s   
ÿ"þr1   c                 C   s|   t  tj | ¡d| ›d¡ t| ƒ}| ¡ }W d  ƒ n1 s!w   Y  d}t ||¡}t  |dud| ›¡ | 	d¡S )zäRead `gitdir` from `.git` file.

    Reads `.git` file in a worktree or submodule and parse `gitdir`.

    Args:
        gitroot (str): Path ending with `.git` file.

    Returns:
        str: Path contained in `.git` file.
    z	Expected z to be a fileNz^gitdir:\s*(.*)$z"Could parse gitdir path from file é   )
r   r6   r	   r&   Úisfiler*   ÚreadlineÚreÚmatchÚgroup)r4   r3   ÚlineZgitdir_patternÚmr   r   r   r:   ¥   s   

ÿ
r:   c                 C   s0   d}t  || ¡}t |dud| ›¡ | d¡S )a  Parse `.git` path.

    Take a path e.g. `/storage/cime/.git/worktrees/cime` and parse `.git`
    directory e.g. `/storage/cime/.git`.

    Args:
        gitdir (str): Path containing `.git` directory.

    Returns:
        str: Path ending with `.git`
    z^(.*/\.git).*NzCould not parse .git from path rF   )rI   rJ   r   r6   rK   )r<   Zdot_git_patternrM   r   r   r   r;   ¾   s   
r;   c                 C   sˆ   t j |d¡}t j |d | ¡¡}t j |¡rt  |¡ t j |¡rBt |d¡}|j	|dd W d   ƒ d S 1 s;w   Y  d S d S )NÚ
SourceModsúSourceMods.{}.tar.gzúw:gz©Úarcname)
r	   r&   r'   r(   r-   r.   r7   Útarfiler*   Úadd)r   r   r   Z
sourcemodsZsourcemods_provÚtfdr   r   r   r   Ó   s   
"ÿÿr   c                 C   sB   t j |d | ¡¡}t j |¡rt  |¡ | d¡}| |¡ d S )Nzbuild_environment.{}.txtÚmach_specific)r	   r&   r'   r(   r-   r.   Úget_envÚsave_all_env_info)r   r   r   Zenv_provÚ
env_moduler   r   r   r   Þ   s
   

r   c                 C   sV   t j |d | ¡¡}t j |¡rt  |¡ d || ¡}t |¡}|r)t||ƒ d S d S )Nzbuild_times.{}.txtz{}/*bldlog*{}.gz)r	   r&   r'   r(   r-   r.   ÚglobÚ_extract_times)r   r   Úbuild_timesÚglobstrÚmatchesr   r   r   r   ç   s   

ÿr   c                 C   s  d}d}| D ]Y}t  d |¡¡\}}}|dkr9| ¡ D ]}| ¡ }|r8| ¡ }	|	d |	d }
}|d |
|¡7 }qt  d |¡¡\}}}|dkr_| ¡ D ]}| ¡ }|r^|t| ¡ d	 ƒ7 }qLqt|d
ƒ}| |¡ | d t	|ƒ¡¡ W d   ƒ d S 1 s€w   Y  d S )NzTarget Build_time
g        zzgrep 'built in' {}r   rF   éþÿÿÿz{} {}
zzgrep -E '^real [0-9.]+$' {}éÿÿÿÿr   zTotal_Elapsed_Time {})
r   rD   r(   Ú
splitlinesÚstripÚsplitÚfloatr*   r+   Ústr)ZzipfilesZtarget_fileÚcontentsZtotal_build_timeÚzipfileÚstatrE   Ú_rL   ÚitemsÚtargetZthe_timer3   r   r   r   r[   ò   s.   €€
"þr[   c                 C   s€   dD ];}d  ||| ¡}t |¡}t t|ƒdk d  |¡¡ |r=|d }| d  | ¡d¡}tj |¡r7t 	|¡ t 
||¡ qd S )N)ZGIT_DESCRIBEÚGIT_LOGS_HEADÚGIT_SUBMODULE_STATUSÚ
GIT_STATUSÚGIT_DIFFÚGIT_LOGÚ
GIT_CONFIGÚ
GIT_REMOTErN   Zbuild_environmentr\   z	{}/{}.{}*é   z5Multiple matches for glob {} should not have happenedr   ú.{}Ú )r(   rZ   r   r6   ÚlenÚreplacer	   r&   r-   r.   Úsymlink)r   r   Úitemr]   r^   Ú	the_matchZgeneric_namer   r   r   r     s   

þ
€èr   c              	      sD  t  ¡ ” |d u rtjd n|}|  d¡}tj |  d¡d¡}tj |¡s*t 	|¡ | 
tj |d |¡¡¡ |  d¡}tj |d¡}|› d|› }tj |¡rUt |¡ t|d	ƒ‰ |  ‡ fd
d„d ¡ t  ||¡ W d   ƒ n1 suw   Y  t| ƒ |  d¡rt| |ƒ W d   ƒ d S W d   ƒ d S 1 s›w   Y  d S )Nr   rV   r   r   zrun_environment.txt.{}ÚRUNDIRúpreview_run.logÚ.r   c                    s   ˆ   d | ¡¡S )Nú{}
)r+   r(   )Úx©r3   r   r   Ú<lambda>>  s    z(save_prerun_provenance.<locals>.<lambda>ÚSAVE_TIMING)r   r   r	   r
   rW   r&   r'   r   r7   ÚmakedirsrX   r(   r-   r.   r*   Úpreview_runr/   Ú_cleanup_spio_statsÚ_record_timing)r   r   rY   ZlogdirZrun_dirZbase_preview_runr„   r   r€   r   Úsave_prerun_provenance)  s0   


ÿ

ü
ä"år‡   c                 C   s”   | d u s
t j | ¡std| › dƒ‚t d | ¡¡ t j | dt 	¡ ||¡}t j 
|¡r3t|› dƒ‚zt  |¡ W |S  tyI   t|› dƒ‚w )NzSAVE_TIMING_DIR zL is not valid. E3SM requires a valid SAVE_TIMING_DIR to archive timing data.z6Archiving timing data and associated provenance in {}.Úperformance_archivezK already exists. Skipping archive of timing data and associated provenance.zN cannot be created. Skipping archive of timing data and associated provenance.)r	   r&   r7   ÚRuntimeErrorÚloggerÚinfor(   r'   ÚgetpassÚgetuserr-   rƒ   ÚOSError)Ú
timing_dirÚ	base_caser   Úfull_timing_dirr   r   r   Ú_check_timing_dirI  s,   
ÿÿÿÿûÿÿr’   c              
   C   s  | j d|  ¡ d}|  |¡sd S |   d¡}|   d¡}|   d¡}|   d¡}|   d¡}|   d¡}zt|||ƒ}	W n tyO }
 zt d	|
¡ W Y d }
~
d S d }
~
ww |   d
¡}|   d¡}t| ƒ}|d urit||||	ƒ t	j
 |d¡}t	j
 |¡r›t t	j
 |	d |¡¡d¡}|j|dd W d   ƒ n1 s–w   Y  t	j
 |	d |¡¡}t	 |¡ t|||||ƒ t||	|ƒ t||	|ƒ t	j
 t	j
 |d¡¡rË|nt	j
 |¡}tjd|d}tt	j
 |	d |¡¡dƒ}| |¡ W d   ƒ n1 sõw   Y  |d ur
t| |||||	ƒ d S d S )NÚPROJECT©Úsubgroupr{   r   r   r   ÚCASEÚSAVE_TIMING_DIRz{!s}ÚMACHÚCOMPILERrN   rO   rP   rQ   zCaseDocs.{}z.gitTr   r   r   )r   Úget_primary_jobÚis_save_timing_dir_projectr’   r‰   rŠ   ÚwarningÚ_get_batch_job_id_for_syslogÚ_record_queue_infor	   r&   r'   r7   rS   r*   r(   rT   ÚmkdirÚ_copy_caseroot_filesÚ_copy_blddir_filesÚ_copy_rundir_filesr-   Údirnamer   r)   r+   Ú_record_syslog)r   r   ÚprojectÚrundirÚblddirr   r   r   r   r‘   ÚeÚmachÚcompilerÚjob_idZsource_mods_dirrU   Ú	case_docsZ	from_repor2   r3   r   r   r   r†   f  sZ   






€ý

ÿý
ÿ
ýÿ
ÿr†   c                 C   s|   | dkrt |||ƒ d S | dv rt|||ƒ d S | dv r$t|||ƒ d S | dv r0t|||ƒ d S | dkr<t|||ƒ d S d S )NÚtheta)úcori-haswellúcori-knl)ÚanvilÚ	chrysalisÚcompy)ÚfrontierÚcrusherÚsummit)Ú_record_anl_theta_queueÚ_record_nersc_queueÚ_record_anl_queueÚ_record_slurm_queueÚ_record_olcf_queue)r©   r«   r   r‘   r   r   r   rž   ¥  s   ÿrž   c                 C   óR   dd|  dfddfD ]\}}d||f }t j|||d t  tj ||¡¡ q
d S )N©zsinfo -a -lÚsinfolúscontrol show jobid %sZ
sqsf_jobid©zOsqueue -o '%.10i %.15P %.20j %.10u %.7a %.2t %.6D %.8C %.10M %.10l %.20S %.20V'Úsqueuef©zsqueue -t R -o '%.10i %R'Zsqueuesú%s.%s©Ú
arg_stdoutrB   ©r   Úrun_cmd_no_failÚgzip_existing_filer	   r&   r'   ©r«   r   r‘   r5   Úfilenamer   r   r   r·   ³  s   
ø
ôr·   c                 C   r»   )Nr¼   r¾   Zscontrol_jobidr¿   rÁ   rÂ   rÃ   rÅ   rÈ   r   r   r   r¹   Ä  s   
ù	õr¹   c                 C   sT   dd  | ¡dfddfD ]\}}d||f }tj|||d t tj ||¡¡ qd S )N)zsinfo -lr½   zsqueue -o '%all' --job {}Zsqueueall_jobid)zMsqueue -o '%.10i %.10P %.15u %.20a %.2t %.6D %.8C %.12M %.12l %.20S %.20V %j'rÀ   rÁ   rÂ   rÃ   )r(   r   rÆ   rÇ   r	   r&   r'   rÈ   r   r   r   r¸   Ô  s   ù	õr¸   c                 C   r»   )N)z¨qstat -l --header JobID:JobName:User:Project:WallTime:QueuedTime:Score:RunTime:TimeRemaining:Nodes:State:Location:Mode:Command:Args:Procs:Queue:StartTime:attrs:GeometryZqstatfzqstat -lf %sZqstatf_jobid)Ú
xtnodestatrÊ   )ÚxtprocadminrË   rÂ   rÃ   rÅ   rÈ   r   r   r   r¶   ã  s   
ù	õr¶   c                 C   s^   ddd|  dffD ]#\}}|d | }t j|d | |d t  tj ||d | ¡¡ q	d S )N)zbjobs -u all >Z
bjobsu_all)z,bjobs -r -u all -o 'jobid slots exec_host' >Zbjobsru_allozbjobs -l -UF %s >ZbjobslUF_jobidú r}   rA   rÅ   )r«   r   r‘   r5   rÉ   Zfull_cmdr   r   r   rº   ò  s   
ýùrº   c                 C   sˆ   |   d¡}t tj |d¡¡D ]}t |¡ qtj |d¡}tj |¡r)t |¡ zt 	|¡ W d S  t
yC   t d |¡¡ Y d S w )Nr{   úio_perf_summary*Ú
spio_statszT{} could not be created. Scorpio I/O statistics will be stored in the run directory.)r   rZ   r	   r&   r'   r.   r-   ÚshutilÚrmtreerƒ   rŽ   rŠ   rœ   r(   )r   r¦   ry   Úspio_stats_dirr   r   r   r…   ý  s   

ÿ
ÿÿr…   c                    sR   ddddddddd	d
  | ¡d
  |¡d  | |¡dg}t ‡ fdd„|D ƒ||¡ d S )Nz
CaseDocs/*zrun_script_provenance/*z*.runz.*.runz*.xmlz	user_nl_*z*env_mach_specific*zMacros*zREADME.casez
Depends.{}zDepends.{}.{}zsoftware_environment.txtc                    ó   g | ]	}t j ˆ |¡‘qS r   ©r	   r&   r'   ©Ú.0r   ©r   r   r   Ú
<listcomp>!  ó    z(_copy_caseroot_files.<locals>.<listcomp>)r(   r   Ú
copy_globs)r©   rª   r   r¬   r   Úglobs_to_copyr   rÖ   r   r      s   
ó r    c                    s(   g d¢}t  ‡ fdd„|D ƒ||¡ d S )N)	rl   rn   ro   rp   rr   rq   rm   zbuild_environment.txtzbuild_times.txtc                    rÒ   r   rÓ   rÔ   ©r§   r   r   r×   3  rØ   z&_copy_blddir_files.<locals>.<listcomp>©r   rÙ   )r§   r‘   r   Zblddir_globs_to_copyr   rÛ   r   r¡   $  s   ÿr¡   c                    s&   dg}t  ‡ fdd„|D ƒ||¡ d S )Nr|   c                    rÒ   r   rÓ   rÔ   ©r¦   r   r   r×   =  rØ   z&_copy_rundir_files.<locals>.<listcomp>rÜ   )r¦   r‘   r   Zrundir_globs_to_copyr   rÝ   r   r¢   7  s
   ÿÿr¢   c              
   C   s€  t j |d |¡¡}t j |¡razJz&t|dƒ}t| ¡  ¡ ƒ}W d   ƒ n1 s*w   Y  t  	|t
j¡ W n ttfyS }	 zt d |	¡¡ W Y d }	~	nd }	~	ww W t  |¡ nt  |¡ w |  d¡}
|
dkr¾t j |d |¡¡}t  |¡ t d ||¡¡ tjdj|
||||d	t j |d
¡d}tt j |d |¡¡dƒ}| d |¡¡ W d   ƒ d S 1 s·w   Y  d S d S )Núsyslog_jobid.{}ÚrúFailed to kill syslog: {}ZSYSLOG_Nr   zcheckpoints.{}z{}/e3sm.log.{}za./mach_syslog {si} {jobid} {lid} {rundir} {rundir}/timing/checkpoints {ac} >& /dev/null & echo $!)ÚsiÚjobidr   r¦   ZacÚToolsrA   r   r~   )r	   r&   r'   r(   r-   r*   ÚintÚreadrb   ÚkillÚsignalÚSIGTERMÚ
ValueErrorrŽ   rŠ   rœ   r.   r   rŸ   r   ÚtouchrÆ   r+   )r   r   r«   r   r¦   r‘   Úsyslog_jobid_pathr3   Úsyslog_jobidr¨   Zsample_intervalZarchive_checkpointsr   r   r   r¤   A  sD   ÿ€ÿ€

ÿ
ûø
"ÿðr¤   c              
   C   sü  t  ¡ ð |d u rtjd n|}|  d¡rä|  d¡}|  d¡}t||ƒ}t||ƒ t||ƒ t||ƒ t  	tj
 |dd| ¡¡ d| }t  tj
 |d|¡¡ | jd|  ¡ d	}|  |¡sf	 W d   ƒ d S |  d
¡}|d u sutj
 |¡s~	 W d   ƒ d S |  d¡}|  d¡}	tj
 |dt ¡ |	|¡}
tj
 |
¡s£	 W d   ƒ d S t| ƒ}|d ur°t||ƒ t jd| |
dd t| ||||||
|ƒ t |
¡D ]!\}}}|D ]}| d¡sât  	tj
 ||¡¡ qÑqÊW d   ƒ d S W d   ƒ d S 1 s÷w   Y  d S )Nr   r‚   r   r{   Útimingze3sm_timing_stats.%sztiming.%s.savedr“   r”   r—   r˜   r–   rˆ   ú	%s.tar.gzFr    ú.gz)r   r   r	   r
   r   Ú_archive_timingsÚ_archive_atm_costsÚ_archive_memory_profileÚ_archive_spio_statsrÇ   r&   r'   rê   rš   r›   r7   rŒ   r   r   Ú_kill_mach_syslogr/   Ú_copy_performance_archive_filesÚwalkÚendswith)r   r   r   r¦   Úrundir_timing_dirÚtiming_saved_filer¥   r   r©   r   r‘   r«   Úrootri   ÚfilesrÉ   r   r   r   Úsave_postrun_provenanced  sp   







ÿ
ç
ã

ÿÚ)

ÿø
€þÄý>"Ârü   c                 C   s|   t j |d|  ¡}t t j |d¡|¡ t d| d¡}|j|t j |¡d W d   ƒ n1 s2w   Y  t 	|¡ |S )Nztiming.rí   rî   rP   rQ   )
r	   r&   r'   rÏ   ÚmoverS   r*   rT   ÚbasenamerÐ   )r   r¦   rø   rU   r   r   r   rð   ©  s   ÿ
rð   c                 C   sL   t j |d¡}t j |¡r$t j |d | ¡¡}t ||¡ t |¡ d S d S )Nzatm_chunk_costs.txtzatm_chunk_costs.{})	r	   r&   r'   r-   r(   rÏ   rý   r   rÇ   )r   r¦   Zatm_chunk_costs_src_pathZatm_chunk_costs_dst_pathr   r   r   rñ   µ  s   
ÿûrñ   c                 C   s`   d}t   tj ||¡¡D ]!}tj tj |¡tj |¡d  | ¡¡}t ||¡ t	 
|¡ qd S )Nzmemory.[0-4].*.logrt   )rZ   r	   r&   r'   r£   rþ   r(   rÏ   rý   r   rÇ   )r   r¦   Úglob_to_copyry   Zmprof_dst_pathr   r   r   rò   ¿  s   ÿûrò   c                 C   s¼   t j |d¡}t j |¡st  |¡ t t j |d¡¡D ]}t ||¡ qt j |d|  ¡}t 	||¡ t
 d| d¡}|j|t j |¡d W d   ƒ n1 sRw   Y  t |¡ d S )NrÎ   rÍ   zspio_stats.rî   rP   rQ   )r	   r&   r'   r-   rŸ   rZ   r   r/   rÏ   ÚcopytreerS   r*   rT   rþ   rÐ   )r   r¦   rÑ   ry   Zspio_stats_job_dirrU   r   r   r   ró   Ê  s   
ÿró   c              
   C   sØ   t j |d | ¡¡}t j |¡rjzSz&t|dƒ}t| ¡  ¡ ƒ}W d   ƒ n1 s*w   Y  t  	|t
j¡ W n ttfyS } zt d |¡¡ W Y d }~nd }~ww W t  |¡ d S W t  |¡ d S t  |¡ w d S )NrÞ   rß   rà   )r	   r&   r'   r(   r-   r*   rä   rå   rb   ræ   rç   rè   ré   rŽ   rŠ   rœ   r.   )r«   r¦   rë   r3   rì   r¨   r   r   r   rô   Û  s    ÿ€ÿÿþørô   c                 C   sÈ  g }|d urC|dv r|  d|  d¡|f ¡ n,|dkr1|  d| ¡ |  d| ¡ |  d| ¡ n|dkrC|  d	| ¡ |  d
| ¡ |  d |¡¡ |  tj |d |¡¡¡ |  tj |d |¡¡¡ |  tj |d |¡¡¡ |  tj |d |¡¡¡ |  d |¡¡ |  d¡ |  tj |d |¡¡¡ |  tj |d¡¡ |D ]<}	t tj ||	¡¡D ]/}
tj |
¡}||krà||vrØ| d¡sØt	j
|
tj |d ||¡¡dd q±t	j
|
|dd q±q¥d S )N)r°   r±   r²   r®   r¯   z	run*%s*%sr–   r­   z%s*errorz	%s*outputz%s*cobaltlogrµ   ze3sm.stderr.%sze3sm.stdout.%szlogs/run_environment.txt.{}ze3sm.log.{}.gzzcpl.log.{}.gzzatm_chunk_costs.{}.gzzmemory.[0-4].*.log.{}.gzztiming/*.{}*Ú
CaseStatuszspio_stats.{}.tar.gzz	replay.shrï   z{}.{}Fr    )Úappendr   r(   r	   r&   r'   rZ   rþ   r÷   r   r/   )r   r   r«   r©   r¦   r   r‘   rù   rÚ   rÿ   ry   rþ   r   r   r   rõ   è  sD   
ý€öÿrõ   c                 C   sb   |   d¡}z!|dv rtjd W S |dv rtjd W S |dv r$tjd W S W dS  ty0   Y dS w )	z4
    mach_syslog only works on certain machines
    r˜   )
r°   r±   r²   r®   r¯   zpm-cpuzpm-gpuZalvarezr³   r´   ÚSLURM_JOB_ID)r­   ÚCOBALT_JOBID)rµ   Ú	LSB_JOBIDN)r   r	   r
   ÚKeyError)r   r©   r   r   r   r     s   
ÿýýr   r>   )1r	   rS   rZ   rI   ÚloggingrŒ   rç   rÏ   Ú
contextlibr   ÚCIMEr   Ú	getLoggerÚ__name__rŠ   r   r   r,   r8   r1   r:   r;   r   r   r   r[   r   r‡   r’   r†   rž   r·   r¹   r¸   r¶   rº   r…   r    r¡   r¢   r¤   rü   rð   rñ   rò   ró   rô   rõ   r   r   r   r   r   Ú<module>   s\    

23
	
 ?

#E
'