{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "baf98b25",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "import h5py \n",
    "import yaml\n",
    "import datetime\n",
    "import os\n",
    "import h5py\n",
    "from tqdm import tqdm\n",
    "from sklearn.decomposition import PCA\n",
    "import json\n",
    "\n",
    "datadir = '/global/cfs/projectdirs/dune/www/data/LBL/SingleCube/Nov2023/ramp/'\n",
    "configdir = '/global/cfs/projectdirs/dune/www/data/LBL/SingleCube/Nov2023/asic_configs/'\n",
    "\n",
    "def get_list_of_h5_files(file_or_dir_name):\n",
    "    if os.path.isfile(file_or_dir_name):\n",
    "        return [file_or_dir_name]\n",
    "    else:\n",
    "        base = file_or_dir_name\n",
    "        sorted_files = [base + \"/\" + name for name in os.listdir(base) if (name[-3:] == \".h5\")]\n",
    "        sorted_files.sort(key=lambda x: os.path.getmtime(x))\n",
    "        return sorted_files\n",
    "\n",
    "# CONSTRUCT Geometry Information for Charge\n",
    "\n",
    "nonrouted_channels=[6,7,8,9,22,23,24,25,38,39,40,54,55,56,57]\n",
    "routed_channels=[i for i in range(64) if i not in nonrouted_channels]\n",
    "\n",
    "geo=None\n",
    "geometrypath = 'layout-2.4.0.yaml'\n",
    "with open(geometrypath) as fi:\n",
    "    geo=yaml.full_load(fi)\n",
    "\n",
    "chip_pix=dict([(chip_id, pix) for chip_id,pix in geo['chips']])\n",
    "\n",
    "N_CHIPS=100\n",
    "N_CHANNELS=64\n",
    "all_uniques = np.zeros( N_CHIPS*N_CHANNELS )\n",
    "pixel_x     = np.zeros( N_CHIPS*N_CHANNELS )\n",
    "pixel_y     = np.zeros( N_CHIPS*N_CHANNELS )\n",
    "\n",
    "def chip_channel_to_index(chip, channel):\n",
    "    return (chip-11)*N_CHANNELS + channel\n",
    "\n",
    "counter_used = set()\n",
    "\n",
    "for chip in range(11, 111):\n",
    "    for channel in range(64):\n",
    "        pix=chip_pix[chip][channel]\n",
    "        counter = chip_channel_to_index(chip, channel)\n",
    "        if not pix is None:\n",
    "            pixel_x[counter] = geo['pixels'][pix][1]\n",
    "            pixel_y[counter] = geo['pixels'][pix][2]\n",
    "\n",
    "#Construct Light Geometry\n",
    "\n",
    "light_channels = [30, 20, 18, 16, 14, 12, 4, 2, 0, 62, 52, 50, 48, 46, 44, 32]\n",
    "light_channel_index = np.ones(N_CHANNELS)*-1\n",
    "\n",
    "for ich, channel in enumerate(light_channels):\n",
    "    light_channel_index[channel] = ich\n",
    "\n",
    "#Gives TOP to BOTTOM numbering of channels\n",
    "light_channel_index = light_channel_index.astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a5315d7b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sync_packet_type = 6\n",
    "vd = 0.016 #cm/clock cycle\n",
    "\n",
    "def packets_to_full_timestamp(packets):\n",
    "    syncs = np.where(packets['packet_type']==6)[0]\n",
    "    correction = np.zeros(len(packets))\n",
    "    for i in tqdm(range(1, len(syncs))):\n",
    "        correction[ syncs[i] :   ] += 1e7\n",
    "    return packets['timestamp'] + correction\n",
    "        \n",
    "def find_events(packets, charge=True, light=True): \n",
    "    \n",
    "    #get true timestamps accounting for sync\n",
    "    true_timestamps = packets_to_full_timestamp(packets)\n",
    "    \n",
    "    no_sync_mask = true_timestamps % 1e7 > 0\n",
    "    packets = packets[no_sync_mask] \n",
    "    true_timestamps = true_timestamps[no_sync_mask]\n",
    "    \n",
    "    dmask = np.logical_and(packets['packet_type']==0, packets['valid_parity']==1)\n",
    "    \n",
    "    dtime, data = true_timestamps[dmask], packets[dmask]\n",
    "    \n",
    "    charge_mask = data['io_channel'] <  5\n",
    "    light_mask  = data['io_channel'] == 5\n",
    "    \n",
    "    ctime, cdata = dtime[charge_mask], data[charge_mask]\n",
    "    ltime, ldata = dtime[light_mask], data[light_mask]\n",
    " \n",
    "    charge_clusters = (np.ones( len(cdata)  )*-1).astype(int)\n",
    "    light_clusters =  (np.ones( len(ldata)  )*-1).astype(int)\n",
    "    \n",
    "    ievt = 0\n",
    "    \n",
    "    #cluster light\n",
    "    \n",
    "    if light:\n",
    "        used = set()\n",
    "        timestamps = set(ltime)\n",
    "        \n",
    "        for timestamp in tqdm(timestamps):\n",
    "            if timestamp in used: continue\n",
    "            td = np.logical_and( ltime < timestamp+5, ltime > timestamp-5 )\n",
    "        \n",
    "            if np.sum(td) < 2: continue\n",
    "        \n",
    "            used.add(timestamp)\n",
    "            used.add(timestamp-1)\n",
    "            used.add(timestamp-2)\n",
    "            used.add(timestamp+1)\n",
    "            used.add(timestamp+2)\n",
    "            used.add(timestamp+3)\n",
    "            used.add(timestamp+4)\n",
    "            used.add(timestamp-3)\n",
    "            used.add(timestamp-4)\n",
    "                \n",
    "            light_clusters[td] = ievt\n",
    "            ievt += 1\n",
    "    \n",
    "    #t0 for each light event\n",
    "    evt_time = np.zeros(ievt)\n",
    "    for evt in set(light_clusters):\n",
    "        time = np.min( ltime[ light_clusters == evt ] )\n",
    "        evt_time[evt] = time\n",
    "        \n",
    "    if charge and light:\n",
    "        timestamps = set(ctime)\n",
    "        used = ctime < 0\n",
    "        for timestamp in tqdm(timestamps):\n",
    "            cdiff = ctime - timestamp\n",
    "            cmask = np.absolute(cdiff) < 2000\n",
    "            if not np.any( np.logical_and( np.logical_not(used), cmask  ) ): continue\n",
    "           \n",
    "            used = np.logical_or(used, cmask)\n",
    "            \n",
    "            #see if any light triggers in this time\n",
    "            min_time = min(ctime[cmask])\n",
    "            \n",
    "            l_ev_diff = -1*(evt_time - min_time)\n",
    "            lmask = np.logical_and(l_ev_diff < 1000, l_ev_diff > -1000)\n",
    "            \n",
    "            if np.any(lmask):\n",
    "                evt = np.max( np.where(lmask) )\n",
    "                charge_clusters[cmask] = evt\n",
    "            else:\n",
    "                charge_clusters[cmask] = ievt\n",
    "                ievt +=1    \n",
    "    \n",
    "    return dict( {\n",
    "        'charge_data'     : cdata,\n",
    "        'charge_time'     : ctime,\n",
    "        'light_data'      : ldata,\n",
    "        'light_time'      : ltime,\n",
    "        'charge_clusters' : charge_clusters,\n",
    "        'light_clusters'  : light_clusters,\n",
    "    } )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b4d6ccd6",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Compiling asic configs...: 100%|██████████| 4666/4666 [00:23<00:00, 201.25it/s]\n",
      "Associating asic configs to date files...: 100%|██████████| 663/663 [00:07<00:00, 92.16it/s]\n"
     ]
    }
   ],
   "source": [
    "def get_date_datafile(filename):\n",
    "    dtime = ((filename.split('packets-')[-1]).split('.h5')[0]).split('_')\n",
    "    year = int(dtime[0])\n",
    "    month = int(dtime[1])\n",
    "    day = int(dtime[2])\n",
    "    hour = int(dtime[3])\n",
    "    minute = int(dtime[4])\n",
    "    second = int(dtime[5])\n",
    "    return year, month, day, hour, minute, second \n",
    "\n",
    "def get_date_config(filename):\n",
    "    dtime = (filename.split('asic_configs_')[-1]).split('_')\n",
    "    year = int(dtime[0])\n",
    "    month = int(dtime[1])\n",
    "    day = int(dtime[2])\n",
    "    hour = int(dtime[3])\n",
    "    minute = int(dtime[4])\n",
    "    second=0\n",
    "    try:\n",
    "        second = int(dtime[-1])\n",
    "    except:\n",
    "        second = 0\n",
    "    return year, month, day, hour, minute, second \n",
    "\n",
    "\n",
    "config_dict = {}\n",
    "pbar = tqdm(os.listdir(configdir))\n",
    "pbar.set_description('Compiling asic configs...')\n",
    "for f in pbar:\n",
    "    if not os.path.isdir(configdir+f) or not ('asic_configs' in f): continue\n",
    "    year, month, day, hour, minute, second  = get_date_config(f)\n",
    "    \n",
    "    larpixconf =   {}\n",
    "    lightpixconf = {}\n",
    "    \n",
    "    \n",
    "    bias_times = {\n",
    "        '45V' : list([datetime.datetime(year=year,day=5, month=12, hour=15, minute=30),\n",
    "                datetime.datetime(year=year,day=5, month=12, hour=19, minute=4)]),\n",
    "        '46V' : list([datetime.datetime(year=year,day=5, month=12, hour=19, minute=4),\n",
    "                 datetime.datetime(year=year, day=6, month=12, hour=23, minute=59)])  \n",
    "    }\n",
    "    \n",
    "    HV_times = {\n",
    "        '0kV' : list([datetime.datetime(year=2019,day=5, month=12, hour=15, minute=30),\n",
    "                datetime.datetime(year=2023,day=5, month=12, hour=16, minute=30)]),\n",
    "        '6kV' : list([datetime.datetime(year=2023,day=5, month=12, hour=16, minute=30),\n",
    "                 datetime.datetime(year=2023, day=5, month=12, hour=16, minute=53)]),\n",
    "        '9kV' : list([datetime.datetime(year=2023,day=5, month=12, hour=16, minute=53),\n",
    "                 datetime.datetime(year=2023, day=5, month=12, hour=16, minute=57)]),\n",
    "        '11kV' : list([datetime.datetime(year=2023,day=5, month=12, hour=16, minute=57),\n",
    "                 datetime.datetime(year=2023, day=5, month=12, hour=17, minute=22)]),\n",
    "        '13kV' : list([datetime.datetime(year=2023,day=5, month=12, hour=17, minute=22),\n",
    "                 datetime.datetime(year=2023, day=5, month=12, hour=17, minute=38)]),\n",
    "        '15kV' : list([datetime.datetime(year=2023,day=5, month=12, hour=18, minute=00),\n",
    "                 datetime.datetime(year=2023, day=6, month=12, hour=23, minute=59)]),\n",
    "    }\n",
    "    \n",
    "    \n",
    "    HV = 'None'\n",
    "    charge = 'None'\n",
    "    light = 'None'\n",
    "\n",
    "    if os.path.exists(configdir+f+'/config_1-1-11.json'):\n",
    "        with open(configdir+f+'/config_1-1-11.json', 'r') as ff:\n",
    "            larpixconf = json.load(ff)\n",
    "        tg = larpixconf['threshold_global']\n",
    "        pr = larpixconf['enable_periodic_trigger']\n",
    "        \n",
    "        if tg==31 and pr==0:\n",
    "            charge='nominal threshold'\n",
    "        elif tg==30 and pr==0:\n",
    "            charge='low threshold'\n",
    "        elif tg==255 and pr==1:\n",
    "            charge='pedestal'\n",
    "        else:\n",
    "            charge='Check logbook'\n",
    "            \n",
    "        dt = datetime.datetime(year=year, day=day, month=month, hour=hour, minute=minute)\n",
    "        for bias in HV_times:\n",
    "            if dt > HV_times[bias][0] and dt < HV_times[bias][1]:\n",
    "                HV = bias\n",
    "        \n",
    "    if os.path.exists(configdir+f+'/config_1-5-11.json'):\n",
    "        with open(configdir+f+'/config_1-5-11.json', 'r') as ff:\n",
    "            lightpixconf = json.load(ff)\n",
    "        \n",
    "        tg = lightpixconf['threshold_global']\n",
    "        if tg==30:\n",
    "            light='nominal threshold'\n",
    "        if tg==29:\n",
    "            light='low threshold'\n",
    "        else:\n",
    "            light='scan'\n",
    "        \n",
    "        dt = datetime.datetime(year=year, day=day, month=month, hour=hour, minute=minute)\n",
    "        for bias in bias_times:\n",
    "            if dt > bias_times[bias][0] and dt < bias_times[bias][1]:\n",
    "                light += ', {}'.format(bias)\n",
    "        \n",
    "    config_dict[f] = {\n",
    "        'year' : year,\n",
    "        'month': month,\n",
    "        'day'  : day,\n",
    "        'hour' : hour,\n",
    "        'minute':minute,\n",
    "        'second':second,\n",
    "        'charge' : charge,\n",
    "        'light'  : light,\n",
    "        'HV'     : HV\n",
    "    }\n",
    "    \n",
    "# Loop over all files, get date, find associated asic configs\n",
    "# Write dictionary with filename, date/time of asic configs, store as json file\n",
    "\n",
    "data_dict   = {}\n",
    "ifile=0\n",
    "pbar = tqdm(get_list_of_h5_files(datadir))\n",
    "pbar.set_description('Associating asic configs to date files...')\n",
    "writedir='/global/cfs/projectdirs/dune/www/data/LBL/SingleCube/Nov2023/simple_v2/'\n",
    "for file in pbar:\n",
    "    year, month, day, hour, minute, second = get_date_datafile(file)\n",
    "    \n",
    "    timedeltas = [ datetime.timedelta( \\\n",
    "        days=(day-config_dict[key]['day']), \n",
    "        seconds=(second-config_dict[key]['second']),\n",
    "        minutes=(minute-config_dict[key]['minute']),\n",
    "        hours=(hour-config_dict[key]['hour']) \\\n",
    "    ) for key in config_dict.keys() ]\n",
    "    \n",
    "    timedeltas = [ t if t > datetime.timedelta(0.) else datetime.timedelta(999999) for t in timedeltas  ]\n",
    "    \n",
    "    #find associated asic config\n",
    "    config  = list(config_dict.keys())[np.argmin( timedeltas )]\n",
    "    data_dict[ifile] = {\n",
    "        \n",
    "        'filename'  : str(file),\n",
    "        'simple_filename' : str(writedir + 'simple-{}'.format( file.split('packets-')[-1] )),\n",
    "        'date'      : str('{}/{}/{}'.format(month, day, year)),\n",
    "        'time'      : str('{}:{}:{}'.format(hour, minute, second)),\n",
    "        'config'    : str('{}/{}'.format(configdir, config)),\n",
    "        'charge'    : str( config_dict[config]['charge'] ),\n",
    "        'light'     : str( config_dict[config]['light'] ),\n",
    "        'HV'        : str( config_dict[config]['HV'] )\n",
    "    }\n",
    "    \n",
    "    ifile += 1\n",
    "    \n",
    "with open('datafiles_nosync.json', 'w') as f:\n",
    "    json.dump(data_dict, f, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f383b32f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_all_channel_mean(unique, data):\n",
    "    u = np.array(list(set(unique)))\n",
    "    d = np.zeros(len(u))\n",
    "    iun=0\n",
    "    for un in tqdm(u):\n",
    "        mask = unique==un\n",
    "        d[iun] = np.mean(data[mask])\n",
    "        iun+=1\n",
    "        \n",
    "    return u, d\n",
    "\n",
    "def get_all_channel_std(unique, data):\n",
    "    u = np.array(list(set(unique)))\n",
    "    d = np.zeros(len(u))\n",
    "    iun=0\n",
    "    for un in tqdm(u):\n",
    "        mask = unique==un\n",
    "        d[iun] = np.std(data[mask])\n",
    "        iun+=1\n",
    "        \n",
    "    return u, d\n",
    "\n",
    "def get_all_channel_count(unique, data):\n",
    "    u = np.array(list(set(unique)))\n",
    "    d = np.zeros(len(u))\n",
    "    iun=0\n",
    "    for un in tqdm(u):\n",
    "        mask = unique==un\n",
    "        d[iun] = np.sum(mask.astype(int))\n",
    "        iun+=1\n",
    "        \n",
    "    return u, d\n",
    " \n",
    "def display_1d(c,title=None):\n",
    "    fig = plt.figure(figsize=(12, 10) )\n",
    "    ax = fig.add_subplot()\n",
    "    ax.hist(c, bins=50, range=(0, 100))\n",
    "    ax.set_xlabel('dataword [adc]', fontsize=12)\n",
    "    ax.grid()\n",
    "    if not title is None: ax.set_title(title)\n",
    "    plt.show()\n",
    "    \n",
    "def display_2d(x, y, c, title=None, cut=None):\n",
    "    fig = plt.figure(figsize=(12, 10) )\n",
    "    ax = fig.add_subplot()\n",
    "    mask = [True]*len(c)\n",
    "    if not cut is None:\n",
    "        mask = c < cut\n",
    "    \n",
    "    sc = ax.scatter(x[mask], y[mask], c=c[mask], marker='s', s=40)\n",
    "    #ax.axhline(50, color='red')\n",
    "\n",
    "    cbar = plt.colorbar(sc, ax=ax)\n",
    "    cbar.set_label('dataword', rotation=270)\n",
    "    if not title is None: ax.set_title(title)\n",
    "    plt.show()\n",
    "    \n",
    "def display_3d(x, y, z, c, title=None, view=None):\n",
    "    fig = plt.figure(figsize=(12, 10) )\n",
    "    ax = fig.add_subplot(projection='3d')\n",
    "    sc = ax.scatter(x, y, z, c=c, marker='s', s=20, alpha=0.5)\n",
    "    cbar = plt.colorbar(sc, ax=ax)\n",
    "    cbar.set_label('dataword', rotation=270, fontsize=14)\n",
    "    if not view is None:\n",
    "        ax.view_init(*view)\n",
    "    \n",
    "    ax.set_xlabel('x [mm]', fontsize=12)\n",
    "    ax.set_ylabel('y [mm]', fontsize=12)\n",
    "    ax.set_zlabel('z (drift) [mm]', fontsize=12)\n",
    "    ax.set_xlim(-150, 150)\n",
    "    ax.set_ylim(-150, 150)\n",
    "    #ax.set_zlim(0, 100)\n",
    "    #ax.set_zlim(0, 5)\n",
    "    if not title is None: ax.set_title(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0c7514ed",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4774/4774 [00:00<00:00, 16926.56it/s]\n"
     ]
    }
   ],
   "source": [
    "def dword_to_mv(dword):\n",
    "    vref = 223/256*1800.\n",
    "    vcm  = 68/256*1800\n",
    "    return (dword/256.)*(vref-vcm) + vcm\n",
    "\n",
    "#Pedestal File\n",
    "filename = '/global/cfs/projectdirs/dune/www/data/LBL/SingleCube/Nov2023/cold/packets-2023_12_05_14_32_11_PST.h5'\n",
    "f = h5py.File(filename)\n",
    "packets = f['packets']\n",
    "dmask = np.logical_and(packets['packet_type']==0, packets['valid_parity']==1)\n",
    "dmask = np.logical_and(dmask, packets['chip_id']<111)\n",
    "data = packets[dmask]\n",
    "\n",
    "q = data['dataword']\n",
    "indices = chip_channel_to_index(data['chip_id'].astype(int), data['channel_id'].astype(int))\n",
    "pedu, pedmean = get_all_channel_mean(indices, q)\n",
    "peds_ind = np.zeros(100*64)\n",
    "for chip in range(11, 111):\n",
    "    for channel in range(64):\n",
    "        counter = chip_channel_to_index(chip, channel)\n",
    "        if counter in pedu:\n",
    "            peds_ind[counter] = dword_to_mv(pedmean[np.where(pedu==counter)[0][0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53d78ace",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import time\n",
    "gain = 4.522 #mV/ke-\n",
    "while True:\n",
    "    try:\n",
    "        for ifile in list(data_dict.keys()):\n",
    "            #start=time.time()\n",
    "            fname = data_dict[ifile]['simple_filename']\n",
    "            if os.path.exists(fname): \n",
    "                continue\n",
    "            if os.path.getsize(data_dict[ifile]['filename'])  > 5e8: continue\n",
    "            f= h5py.File( data_dict[ifile]['filename'] )\n",
    "            packets = f['packets']\n",
    "            if len(packets) < 10:\n",
    "                continue\n",
    "            evs = find_events(packets)\n",
    "            hf = h5py.File(fname, 'w')\n",
    "            hf.create_dataset('light_ts', data=evs['light_time'])\n",
    "            hf.create_dataset('charge_ts', data=evs['charge_time'])\n",
    "            hf.create_dataset('charge_event', data=evs['charge_clusters'])\n",
    "            hf.create_dataset('light_event', data=evs['light_clusters'])\n",
    "            hf.create_dataset('charge_chipid', data=evs['charge_data']['chip_id'])\n",
    "            hf.create_dataset('charge_chanid', data=evs['charge_data']['channel_id'])\n",
    "    \n",
    "            indices = chip_channel_to_index(evs['charge_data']['chip_id'].astype(int),\n",
    "                                    evs['charge_data']['channel_id'].astype(int))\n",
    "    \n",
    "            hf.create_dataset('charge_q', data= (dword_to_mv(evs['charge_data']['dataword'])-peds_ind[indices])/gain )\n",
    "            hf.create_dataset('charge_x', data=pixel_x[indices] )\n",
    "            hf.create_dataset('charge_y', data=pixel_y[indices] )\n",
    "            hf.create_dataset('light_chanid', data=evs['light_data']['channel_id'])\n",
    "            hf.create_dataset('light_data', data=evs['light_data']['dataword'])\n",
    "            hf.close()\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f1db1756-a5e8-436a-9aee-36bfcadf87e5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4774"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dbdb234d-e9ae-486b-af88-6147750162ca",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "NERSC Python",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
