ROOT logo
#if !defined(__CINT__) || defined(__MAKECINT__)

#include <Riostream.h>

// ROOT includes
#include "TString.h"
#include "TObjArray.h"
#include "TObjString.h"
#include "TFileMerger.h"
#include "TGrid.h"
#include "TList.h"
#include "TSystem.h"
#include "TGridResult.h"
#include "TMap.h"
#include "TFile.h"
#include "TROOT.h"
#endif

Bool_t AddFileList(TString, TString&, TString);
void ReadListFromFile(TString, TString&, TString);
Int_t GetLastStage(TGridResult*);

void mergeGridFiles(TString outFilename, TString inFileList, TString addPrefix = "alien://", Int_t nFilesPerStep = 100, Bool_t copyLocal = kTRUE, TString dirsToMerge = "")
{
  TString fileList = "";
  ReadListFromFile(inFileList, fileList, addPrefix);

  if ( fileList.IsNull() ) {
    printf("List of merging files is null: Nothing done!\n");
    return;
  }

  TString logFilename = "toBeMerged.txt";

  if ( ! gGrid ) copyLocal = kFALSE;

  TString currList = fileList;
  TString currOutput = "", mergedFiles = "";

  Bool_t showProgressBar = ! gROOT->IsBatch();

  for ( Int_t istep = 0; istep < 100; istep++ ) {
    TFileMerger fileMerger(copyLocal);
    Int_t mergeType = ( TFileMerger::kRegular | TFileMerger::kAll );
    if ( ! dirsToMerge.IsNull() ) {
      fileMerger.AddObjectNames(dirsToMerge.Data());
      mergeType |= TFileMerger::kOnlyListed;
    }
    TObjArray* array = currList.Tokenize(" ");
    currList = "";
    Int_t nFiles = array->GetEntries();
    Int_t subStep = -1;
    for (Int_t ifile = 0; ifile < nFiles; ifile++ ) {
      TString currFilename = array->At(ifile)->GetName();
      Bool_t isFileAdded = fileMerger.AddFile(currFilename.Data(), showProgressBar);
      if ( ! isFileAdded ) continue;
      
      Int_t nFilesToMerge = fileMerger.GetMergeList()->GetEntries();
      if ( nFilesToMerge % nFilesPerStep != 0 && ifile < nFiles - 1 ) 
        continue;
      // The following part is executed only at the end of each step
      currOutput = outFilename;
      if ( nFiles > nFilesPerStep ) {
        subStep++;
        currOutput.ReplaceAll(".root",Form("_%i_%i.root", istep, subStep));
        AddFileList(currOutput, currList, "");
      }
      fileMerger.OutputFile(currOutput.Data(),kTRUE,1); // needed when merging single files for specific directories
      fileMerger.PartialMerge(mergeType);
      printf("\nMerged in %s:\n", currOutput.Data());
      mergedFiles = "";
      for ( Int_t ientry=0; ientry<nFilesToMerge; ientry++ )
        mergedFiles += Form("%s ", fileMerger.GetMergeList()->At(ientry)->GetName());
      printf("%s\n\n", mergedFiles.Data());

      // remove merged files
      if ( istep > 0 )
        gSystem->Exec(Form("rm %s", mergedFiles.Data()));

      // Write log file to keep trace of files to be merged
      ofstream logFile(logFilename.Data());
      TString logString = "";
      for ( Int_t jfile = ifile + 1; jfile < nFiles; jfile++ ) {
        logString += Form("%s ", array->At(jfile)->GetName());
      }
      logString.Append(currList.Data());
      logString.ReplaceAll(" ", "\n");
      logFile << logString.Data() << endl;;
      logFile.close();
    } // loop on files

    delete array;
    printf("Step %i completed!\n", istep);

    if ( nFiles <= nFilesPerStep ) break;
  } // loop on steps

  gSystem->Exec(Form("rm %s", logFilename.Data()));
}


//___________________________________________________
Bool_t AddFileList(TString filename, TString& fileList, TString addPrefix)
{
  if ( filename.IsNull() || ! filename.Contains(".root") ) return kFALSE;

  if ( ! addPrefix.IsNull() && ! filename.Contains(addPrefix.Data()) )
    filename.Prepend(addPrefix.Data());

  if ( filename.Contains("alien://") && ! gGrid )
    TGrid::Connect("alien://");
  
  if ( ! fileList.IsNull() )
    fileList.Append(" ");
  fileList.Append(filename.Data());

  return kTRUE;
}

//___________________________________________________
void ReadListFromFile(TString filename, TString& fileList, TString addPrefix)
{
  ifstream inFile(filename.Data());
  TString currLine = "";
  if ( inFile.is_open() ) {
    while ( ! inFile.eof() ) {
      currLine.ReadLine(inFile,kTRUE); // Read line
      AddFileList(currLine, fileList, addPrefix);
    }
    inFile.close();
  }
}


//___________________________________________________
void completeProd(TString runListName="runList.txt", TString prodDir = "", TString baseDir="/alice/data/2010/LHC10h", TString outTaskFilename="QAresults.root", Int_t nFilesPerStep = 50, TString dirsToMerge = "MUON_QA MTR_ChamberEffMap", Bool_t mergeFast = kFALSE, Bool_t overwriteExisting = kFALSE)
{
  TString outFilename = "completeFileList.txt";

  // Get run list from file
  ifstream inFile(runListName.Data());
  TObjArray runList;
  runList.SetOwner();
  TString currRun;
  Int_t minRun = 99999999;
  Int_t maxRun = -1;
  if (inFile.is_open()) {
    while (! inFile.eof() ) {
      currRun.ReadLine(inFile,kTRUE); // Read line
      if ( currRun.IsNull() || ! currRun.IsDigit() ) continue;
      Int_t currRunInt = currRun.Atoi();
      minRun = TMath::Min(currRunInt, minRun);
      maxRun = TMath::Max(currRunInt, maxRun);
      runList.Add(new TObjString(Form("%d", currRunInt)));
    }
    inFile.close();
  }

  outFilename.ReplaceAll(".txt", Form("_%i_%i.txt", minRun, maxRun));

  //TString filePattern[3] = {"", "Stage*/","*/"};
  TString filePattern[2] = {"","*/"};

  ofstream outFile(outFilename.Data());

  const Int_t kNlibs = 5; // 1
  TString loadLibs[kNlibs] = {"libANALYSIS.so", "libOADB.so", "libANALYSISalice.so", "libCORRFW.so", "libPWGmuon.so"};
  for ( Int_t ilib=0; ilib<kNlibs; ilib++ ) {
    Int_t exitVal = gSystem->Load(loadLibs[ilib].Data());
    if ( exitVal < 0 ) {
      printf("Please run with aliroot if you're merging QA objects!\n");
      return;
    }
  }

  if ( ! gGrid )
    TGrid::Connect("alien://");

  baseDir.ReplaceAll("alien://","");

  TMap* map = 0x0;
  TString stageName = "";
  TString runsWithoutOut = "";
  for ( Int_t irun=0; irun<runList.GetEntries(); irun++ ) {
    TString currRunString = ((TObjString*)runList.At(irun))->GetString();

    TString localOut = outTaskFilename;
    localOut.ReplaceAll(".root", Form("_%s.root", currRunString.Data()));
    if ( ! gSystem->AccessPathName(localOut.Data()) ) {
      if ( overwriteExisting )
        printf("Overwriting existing file %s\n", localOut.Data());
      else {
        printf("Warning: merged file %s already exist: do not overwrite\n", localOut.Data());
        outFile << gSystem->pwd() << "/" << localOut.Data() << endl;
        continue;
      }
    }

    TString tmpFilename = Form("%s/tmp_mergeListRun%s.txt", gSystem->pwd(), currRunString.Data());
    TString mergeFilename = "";

    Int_t nPatterns = ( mergeFast ) ? 1 : 2;
    
    for ( Int_t ipattern=0; ipattern<nPatterns; ipattern++ ) {
      TString command = ( prodDir.Contains("private") ) ? Form("find %s/ *%s/%s%s", baseDir.Data(), currRunString.Data(), filePattern[ipattern].Data(), outTaskFilename.Data()) : Form("find %s/*%s /%s/%s%s", baseDir.Data(), currRunString.Data(), prodDir.Data(), filePattern[ipattern].Data(), outTaskFilename.Data());

      printf("%s\n", command.Data());

      TGridResult* res = gGrid->Command(command);

      if ( ! res || res->GetEntries() == 0 ) continue;

      ofstream tmpFile(tmpFilename.Data());
      
      Int_t mergeStage = ( ipattern == 1 ) ? GetLastStage(res)  : -1;
      stageName = Form("Stage_%i", mergeStage);

      TIter nextmap(res);
      while ( ( map = (TMap*)nextmap() ) ) {
        // Loop 'find' results and get next LFN
        TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
        if (!objs || !objs->GetString().Length()) 
          continue;

        mergeFilename = objs->GetString();

        if ( mergeStage > 0 && ! mergeFilename.Contains(stageName.Data()) ) continue;
        if ( mergeFilename.Contains(".resubmit") ) continue; // Avoid double counting for runs where a resubmission was performed

        tmpFile << mergeFilename.Data() << endl;
      } // loop on grid lfns

      delete res;

      tmpFile.close();

      // If the merging of specific directories is required
      // run merging also if there is 1 file
      // so that we get rid of other sub-directories
      if ( ipattern == 1 || ! dirsToMerge.IsNull() ) {
        mergeFilename = outTaskFilename;
        mergeFilename.ReplaceAll(".root", Form("_%s.root", currRunString.Data()));
        mergeGridFiles(mergeFilename, tmpFilename, "alien://", nFilesPerStep, kTRUE, dirsToMerge);
      }
      
      gSystem->Exec(Form("rm %s", tmpFilename.Data()));

      if ( ! mergeFilename.Contains("alien://") ) outFile << gSystem->pwd() << "/";
      outFile << mergeFilename.Data() << endl;
      break;
    } // loop on pattern
    if ( mergeFilename.IsNull() ) runsWithoutOut += currRunString + " ";
  } // loop on runs
  if ( ! runsWithoutOut.IsNull() )
    printf("\nNo output found in runs\n%s\n",runsWithoutOut.Data());
  printf("\nOutput written in:\n%s\n", outFilename.Data());
}


//___________________________________________________
Int_t GetLastStage(TGridResult* res)
{
  Int_t lastStage = 0;

  TMap* map = 0x0;
  TIter nextmap(res);
  TString filename = "", currToken = "";
  while ( ( map = (TMap*)nextmap() ) ) {
    // Loop 'find' results and get next LFN
    TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
    if (!objs || !objs->GetString().Length()) 
      continue;

    filename = objs->GetString();
    
    if ( ! filename.Contains("Stage_") ) continue;

    TObjArray* array = filename.Tokenize("/");
    array->SetOwner();
    for ( Int_t ientry=0; ientry<array->GetEntries(); ientry++ ) {
      currToken = array->At(ientry)->GetName();
      if ( currToken.Contains("Stage_") ) {
        currToken.Remove(0,6);
        Int_t currStage = currToken.Atoi();
        lastStage = TMath::Max(currStage, lastStage);
        break;
      }
    }
    delete array;
  } // loop on grid lfns

  return lastStage;
}
 mergeGridFiles.C:1
 mergeGridFiles.C:2
 mergeGridFiles.C:3
 mergeGridFiles.C:4
 mergeGridFiles.C:5
 mergeGridFiles.C:6
 mergeGridFiles.C:7
 mergeGridFiles.C:8
 mergeGridFiles.C:9
 mergeGridFiles.C:10
 mergeGridFiles.C:11
 mergeGridFiles.C:12
 mergeGridFiles.C:13
 mergeGridFiles.C:14
 mergeGridFiles.C:15
 mergeGridFiles.C:16
 mergeGridFiles.C:17
 mergeGridFiles.C:18
 mergeGridFiles.C:19
 mergeGridFiles.C:20
 mergeGridFiles.C:21
 mergeGridFiles.C:22
 mergeGridFiles.C:23
 mergeGridFiles.C:24
 mergeGridFiles.C:25
 mergeGridFiles.C:26
 mergeGridFiles.C:27
 mergeGridFiles.C:28
 mergeGridFiles.C:29
 mergeGridFiles.C:30
 mergeGridFiles.C:31
 mergeGridFiles.C:32
 mergeGridFiles.C:33
 mergeGridFiles.C:34
 mergeGridFiles.C:35
 mergeGridFiles.C:36
 mergeGridFiles.C:37
 mergeGridFiles.C:38
 mergeGridFiles.C:39
 mergeGridFiles.C:40
 mergeGridFiles.C:41
 mergeGridFiles.C:42
 mergeGridFiles.C:43
 mergeGridFiles.C:44
 mergeGridFiles.C:45
 mergeGridFiles.C:46
 mergeGridFiles.C:47
 mergeGridFiles.C:48
 mergeGridFiles.C:49
 mergeGridFiles.C:50
 mergeGridFiles.C:51
 mergeGridFiles.C:52
 mergeGridFiles.C:53
 mergeGridFiles.C:54
 mergeGridFiles.C:55
 mergeGridFiles.C:56
 mergeGridFiles.C:57
 mergeGridFiles.C:58
 mergeGridFiles.C:59
 mergeGridFiles.C:60
 mergeGridFiles.C:61
 mergeGridFiles.C:62
 mergeGridFiles.C:63
 mergeGridFiles.C:64
 mergeGridFiles.C:65
 mergeGridFiles.C:66
 mergeGridFiles.C:67
 mergeGridFiles.C:68
 mergeGridFiles.C:69
 mergeGridFiles.C:70
 mergeGridFiles.C:71
 mergeGridFiles.C:72
 mergeGridFiles.C:73
 mergeGridFiles.C:74
 mergeGridFiles.C:75
 mergeGridFiles.C:76
 mergeGridFiles.C:77
 mergeGridFiles.C:78
 mergeGridFiles.C:79
 mergeGridFiles.C:80
 mergeGridFiles.C:81
 mergeGridFiles.C:82
 mergeGridFiles.C:83
 mergeGridFiles.C:84
 mergeGridFiles.C:85
 mergeGridFiles.C:86
 mergeGridFiles.C:87
 mergeGridFiles.C:88
 mergeGridFiles.C:89
 mergeGridFiles.C:90
 mergeGridFiles.C:91
 mergeGridFiles.C:92
 mergeGridFiles.C:93
 mergeGridFiles.C:94
 mergeGridFiles.C:95
 mergeGridFiles.C:96
 mergeGridFiles.C:97
 mergeGridFiles.C:98
 mergeGridFiles.C:99
 mergeGridFiles.C:100
 mergeGridFiles.C:101
 mergeGridFiles.C:102
 mergeGridFiles.C:103
 mergeGridFiles.C:104
 mergeGridFiles.C:105
 mergeGridFiles.C:106
 mergeGridFiles.C:107
 mergeGridFiles.C:108
 mergeGridFiles.C:109
 mergeGridFiles.C:110
 mergeGridFiles.C:111
 mergeGridFiles.C:112
 mergeGridFiles.C:113
 mergeGridFiles.C:114
 mergeGridFiles.C:115
 mergeGridFiles.C:116
 mergeGridFiles.C:117
 mergeGridFiles.C:118
 mergeGridFiles.C:119
 mergeGridFiles.C:120
 mergeGridFiles.C:121
 mergeGridFiles.C:122
 mergeGridFiles.C:123
 mergeGridFiles.C:124
 mergeGridFiles.C:125
 mergeGridFiles.C:126
 mergeGridFiles.C:127
 mergeGridFiles.C:128
 mergeGridFiles.C:129
 mergeGridFiles.C:130
 mergeGridFiles.C:131
 mergeGridFiles.C:132
 mergeGridFiles.C:133
 mergeGridFiles.C:134
 mergeGridFiles.C:135
 mergeGridFiles.C:136
 mergeGridFiles.C:137
 mergeGridFiles.C:138
 mergeGridFiles.C:139
 mergeGridFiles.C:140
 mergeGridFiles.C:141
 mergeGridFiles.C:142
 mergeGridFiles.C:143
 mergeGridFiles.C:144
 mergeGridFiles.C:145
 mergeGridFiles.C:146
 mergeGridFiles.C:147
 mergeGridFiles.C:148
 mergeGridFiles.C:149
 mergeGridFiles.C:150
 mergeGridFiles.C:151
 mergeGridFiles.C:152
 mergeGridFiles.C:153
 mergeGridFiles.C:154
 mergeGridFiles.C:155
 mergeGridFiles.C:156
 mergeGridFiles.C:157
 mergeGridFiles.C:158
 mergeGridFiles.C:159
 mergeGridFiles.C:160
 mergeGridFiles.C:161
 mergeGridFiles.C:162
 mergeGridFiles.C:163
 mergeGridFiles.C:164
 mergeGridFiles.C:165
 mergeGridFiles.C:166
 mergeGridFiles.C:167
 mergeGridFiles.C:168
 mergeGridFiles.C:169
 mergeGridFiles.C:170
 mergeGridFiles.C:171
 mergeGridFiles.C:172
 mergeGridFiles.C:173
 mergeGridFiles.C:174
 mergeGridFiles.C:175
 mergeGridFiles.C:176
 mergeGridFiles.C:177
 mergeGridFiles.C:178
 mergeGridFiles.C:179
 mergeGridFiles.C:180
 mergeGridFiles.C:181
 mergeGridFiles.C:182
 mergeGridFiles.C:183
 mergeGridFiles.C:184
 mergeGridFiles.C:185
 mergeGridFiles.C:186
 mergeGridFiles.C:187
 mergeGridFiles.C:188
 mergeGridFiles.C:189
 mergeGridFiles.C:190
 mergeGridFiles.C:191
 mergeGridFiles.C:192
 mergeGridFiles.C:193
 mergeGridFiles.C:194
 mergeGridFiles.C:195
 mergeGridFiles.C:196
 mergeGridFiles.C:197
 mergeGridFiles.C:198
 mergeGridFiles.C:199
 mergeGridFiles.C:200
 mergeGridFiles.C:201
 mergeGridFiles.C:202
 mergeGridFiles.C:203
 mergeGridFiles.C:204
 mergeGridFiles.C:205
 mergeGridFiles.C:206
 mergeGridFiles.C:207
 mergeGridFiles.C:208
 mergeGridFiles.C:209
 mergeGridFiles.C:210
 mergeGridFiles.C:211
 mergeGridFiles.C:212
 mergeGridFiles.C:213
 mergeGridFiles.C:214
 mergeGridFiles.C:215
 mergeGridFiles.C:216
 mergeGridFiles.C:217
 mergeGridFiles.C:218
 mergeGridFiles.C:219
 mergeGridFiles.C:220
 mergeGridFiles.C:221
 mergeGridFiles.C:222
 mergeGridFiles.C:223
 mergeGridFiles.C:224
 mergeGridFiles.C:225
 mergeGridFiles.C:226
 mergeGridFiles.C:227
 mergeGridFiles.C:228
 mergeGridFiles.C:229
 mergeGridFiles.C:230
 mergeGridFiles.C:231
 mergeGridFiles.C:232
 mergeGridFiles.C:233
 mergeGridFiles.C:234
 mergeGridFiles.C:235
 mergeGridFiles.C:236
 mergeGridFiles.C:237
 mergeGridFiles.C:238
 mergeGridFiles.C:239
 mergeGridFiles.C:240
 mergeGridFiles.C:241
 mergeGridFiles.C:242
 mergeGridFiles.C:243
 mergeGridFiles.C:244
 mergeGridFiles.C:245
 mergeGridFiles.C:246
 mergeGridFiles.C:247
 mergeGridFiles.C:248
 mergeGridFiles.C:249
 mergeGridFiles.C:250
 mergeGridFiles.C:251
 mergeGridFiles.C:252
 mergeGridFiles.C:253
 mergeGridFiles.C:254
 mergeGridFiles.C:255
 mergeGridFiles.C:256
 mergeGridFiles.C:257
 mergeGridFiles.C:258
 mergeGridFiles.C:259
 mergeGridFiles.C:260
 mergeGridFiles.C:261
 mergeGridFiles.C:262
 mergeGridFiles.C:263
 mergeGridFiles.C:264
 mergeGridFiles.C:265
 mergeGridFiles.C:266
 mergeGridFiles.C:267
 mergeGridFiles.C:268
 mergeGridFiles.C:269
 mergeGridFiles.C:270
 mergeGridFiles.C:271
 mergeGridFiles.C:272
 mergeGridFiles.C:273
 mergeGridFiles.C:274
 mergeGridFiles.C:275
 mergeGridFiles.C:276
 mergeGridFiles.C:277
 mergeGridFiles.C:278
 mergeGridFiles.C:279
 mergeGridFiles.C:280
 mergeGridFiles.C:281
 mergeGridFiles.C:282
 mergeGridFiles.C:283
 mergeGridFiles.C:284
 mergeGridFiles.C:285
 mergeGridFiles.C:286
 mergeGridFiles.C:287
 mergeGridFiles.C:288
 mergeGridFiles.C:289
 mergeGridFiles.C:290
 mergeGridFiles.C:291
 mergeGridFiles.C:292
 mergeGridFiles.C:293
 mergeGridFiles.C:294