RUNNING python3 install/spine/bin/run.py --config /tmp/tmp.t5xBRum4FD/ndlar_single_full_chain_250505.cfg --log_dir /pscratch/sd/d/dunepro/cuddandr/logs/FSD_CosmicRun3/run-mlreco/FSD_CosmicRun3.spine/LOGS/0000000 --source /pscratch/sd/d/dunepro/cuddandr/output/FSD_CosmicRun3/run-mlreco/FSD_CosmicRun3.flow2supera/LARCV/0000000/FSD_CosmicRun3.flow2supera.0000069.LARCV.root --output /pscratch/sd/d/dunepro/cuddandr/output/FSD_CosmicRun3/tmp/run-mlreco/FSD_CosmicRun3.spine/FSD_CosmicRun3.spine.0000069.MLRECO_SPINE.hdf5

 ██████████   ██████████    ███   ███       ██   ███████████
███        █  ██       ███   █    █████     ██   ██         
  ████████    ██       ███  ███   ██  ████  ██   ██████████ 
█        ███  ██████████     █    ██     █████   ██         
 ██████████   ██            ███   ██       ███   ███████████

Release version: 0.5.1

$CUDA_VISIBLE_DEVICES=0

Configuration processed at: Linux nid003520 5.14.21-150500.55.97_13.0.78-cray_shasta_c #1 SMP Thu Mar 13 20:09:44 UTC 2025 (330b47d) x86_64 x86_64 x86_64 GNU/Linux

base: {world_size: 1, iterations: -1, seed: 0, dtype: float32, unwrap: true, log_dir: /pscratch/sd/d/dunepro/cuddandr/logs/FSD_CosmicRun3/run-mlreco/FSD_CosmicRun3.spine/LOGS/0000000,
  prefix_log: true, overwrite_log: true, log_step: 1, split_output: false, parent_path: /tmp/tmp.t5xBRum4FD}
io:
  loader:
    batch_size: 16
    shuffle: false
    num_workers: 8
    collate_fn: all
    dataset:
      name: larcv
      file_keys: [/pscratch/sd/d/dunepro/cuddandr/output/FSD_CosmicRun3/run-mlreco/FSD_CosmicRun3.flow2supera/LARCV/0000000/FSD_CosmicRun3.flow2supera.0000069.LARCV.root]
      schema:
        data: {parser: sparse3d, sparse_event: sparse3d_pcluster}
        seg_label: {parser: sparse3d, sparse_event: sparse3d_pcluster_semantics}
        ppn_label: {parser: particle_points, sparse_event: sparse3d_pcluster, particle_event: particle_pcluster,
          include_point_tagging: false}
        clust_label: {parser: cluster3d, cluster_event: cluster3d_pcluster, particle_event: particle_pcluster,
          neutrino_event: neutrino_mc_truth, sparse_semantics_event: sparse3d_pcluster_semantics,
          add_particle_info: true, clean_data: true}
        coord_label: {parser: particle_coords, particle_event: particle_pcluster,
          cluster_event: cluster3d_pcluster}
        particles: {parser: particle, particle_event: particle_pcluster, neutrino_event: neutrino_mc_truth,
          cluster_event: cluster3d_pcluster, skip_empty: true}
        neutrinos: {parser: neutrino, neutrino_event: neutrino_mc_truth, cluster_event: cluster3d_pcluster}
        meta: {parser: meta, sparse_event: sparse3d_pcluster}
        run_info: {parser: run_info, sparse_event: sparse3d_pcluster}
        trigger: {parser: trigger, trigger_event: trigger_base}
        flashes: {parser: flash, flash_event: opflash_light}
  writer:
    name: hdf5
    file_name: /pscratch/sd/d/dunepro/cuddandr/output/FSD_CosmicRun3/tmp/run-mlreco/FSD_CosmicRun3.spine/FSD_CosmicRun3.spine.0000069.MLRECO_SPINE.hdf5
    overwrite: true
    keys: [run_info, meta, trigger, flashes, points, points_label, depositions, depositions_label,
      reco_particles, truth_particles, reco_interactions, truth_interactions]
model:
  name: full_chain
  weight_path: /dvs_ro/cfs/projectdirs/dune/users/drielsma/weights/ndlar_250505_snapshot.ckpt
  to_numpy: true
  network_input: {data: data, seg_label: seg_label}
  modules:
    chain: {deghosting: null, charge_rescaling: null, segmentation: uresnet, point_proposal: ppn,
      fragmentation: graph_spice, shower_aggregation: grappa, shower_primary: grappa,
      track_aggregation: grappa, particle_aggregation: null, inter_aggregation: grappa,
      particle_identification: grappa, primary_identification: grappa, orientation_identification: grappa,
      calibration: null}
    uresnet_ppn:
      uresnet:
        num_input: 1
        num_classes: 5
        filters: 32
        depth: 5
        reps: 2
        allow_bias: false
        activation: {name: lrelu, negative_slope: 0.33}
        norm_layer: {name: batch_norm, eps: 0.0001, momentum: 0.01}
      ppn: {classify_endpoints: false}
    uresnet_ppn_loss:
      uresnet_loss: {balance_loss: false}
      ppn_loss: {mask_loss: CE, resolution: 5.0}
    graph_spice:
      shapes: [shower, track, michel, delta]
      use_raw_features: true
      invert: true
      make_clusters: true
      embedder:
        spatial_embedding_dim: 3
        feature_embedding_dim: 16
        occupancy_mode: softplus
        covariance_mode: softplus
        uresnet:
          num_input: 4
          filters: 32
          input_kernel: 5
          depth: 5
          reps: 2
          spatial_size: 6144
          allow_bias: false
          activation: {name: lrelu, negative_slope: 0.33}
          norm_layer: {name: batch_norm, eps: 0.0001, momentum: 0.01}
      kernel: {name: bilinear, num_features: 32}
      constructor:
        edge_threshold: 0.1
        min_size: 3
        label_edges: false
        graph: {name: radius, r: 1.9}
        orphan: {mode: radius, radius: 1.9, iterate: true, assign_all: true}
    graph_spice_loss: {name: edge, loss: binary_log_dice_ce}
    grappa_shower:
      nodes:
        source: cluster
        shapes: [shower, michel, delta]
        make_groups: true
        grouping_method: score
      graph:
        name: complete
        max_length: [500, 0, 500, 500, 0, 0, 0, 25, 0, 25]
        dist_algorithm: recursive
      node_encoder: {name: geo, use_numpy: true, add_value: true, add_shape: true,
        add_points: true, add_local_dirs: true, dir_max_dist: 5, add_local_dedxs: true,
        dedx_max_dist: 5}
      edge_encoder: {name: geo, use_numpy: true}
      gnn_model:
        name: meta
        node_feats: 33
        edge_feats: 19
        node_pred: 2
        edge_pred: 2
        edge_layer:
          name: mlp
          mlp:
            depth: 3
            width: 64
            activation: {name: lrelu, negative_slope: 0.1}
            normalization: batch_norm
        node_layer:
          name: mlp
          reduction: max
          attention: false
          message_mlp:
            depth: 3
            width: 64
            activation: {name: lrelu, negative_slope: 0.1}
            normalization: batch_norm
          aggr_mlp:
            depth: 3
            width: 64
            activation: {name: lrelu, negative_slope: 0.1}
            normalization: batch_norm
    grappa_shower_loss:
      node_loss: {name: shower_primary, use_closest: true, high_purity: true, use_group_pred: true}
      edge_loss: {name: channel, target: group, high_purity: true}
    grappa_track:
      nodes:
        source: cluster
        shapes: [track]
        make_groups: true
        grouping_method: score
      graph: {name: complete, max_length: 100, dist_algorithm: recursive}
      node_encoder: {name: geo, use_numpy: true, add_value: true, add_shape: false,
        add_points: true, add_local_dirs: true, dir_max_dist: 5, add_local_dedxs: true,
        dedx_max_dist: 5}
      edge_encoder: {name: geo, use_numpy: true}
      gnn_model:
        name: meta
        node_feats: 32
        edge_feats: 19
        edge_pred: 2
        edge_layer:
          name: mlp
          mlp:
            depth: 3
            width: 64
            activation: {name: lrelu, negative_slope: 0.1}
            normalization: batch_norm
        node_layer:
          name: mlp
          reduction: max
          attention: false
          message_mlp:
            depth: 3
            width: 64
            activation: {name: lrelu, negative_slope: 0.1}
            normalization: batch_norm
          aggr_mlp:
            depth: 3
            width: 64
            activation: {name: lrelu, negative_slope: 0.1}
            normalization: batch_norm
    grappa_track_loss:
      edge_loss: {name: channel, target: group}
    grappa_inter:
      nodes:
        source: group
        shapes: [shower, track, michel, delta]
        min_size: -1
        make_groups: true
      graph:
        name: complete
        max_length: [500, 500, 0, 0, 25, 25, 25, 0, 0, 0]
        dist_algorithm: recursive
      node_encoder: {name: geo, use_numpy: true, add_value: true, add_shape: true,
        add_points: true, add_local_dirs: true, dir_max_dist: 5, add_local_dedxs: true,
        dedx_max_dist: 5}
      edge_encoder: {name: geo, use_numpy: true}
      gnn_model:
        name: meta
        node_feats: 33
        edge_feats: 19
        node_pred: {type: 6, primary: 2, orient: 2}
        edge_pred: 2
        edge_layer:
          name: mlp
          mlp:
            depth: 3
            width: 128
            activation: {name: lrelu, negative_slope: 0.1}
            normalization: batch_norm
        node_layer:
          name: mlp
          reduction: max
          attention: false
          message_mlp:
            depth: 3
            width: 128
            activation: {name: lrelu, negative_slope: 0.1}
            normalization: batch_norm
          aggr_mlp:
            depth: 3
            width: 128
            activation: {name: lrelu, negative_slope: 0.1}
            normalization: batch_norm
    grappa_inter_loss:
      node_loss:
        type: {name: class, target: pid, loss: ce, balance_loss: true}
        primary: {name: class, target: inter_primary, loss: ce, balance_loss: true}
        orient: {name: orient, loss: ce}
      edge_loss: {name: channel, target: interaction}
build: {mode: both, units: cm, fragments: false, particles: true, interactions: true}
post:
  shape_logic: {enforce_pid: true, enforce_primary: true, priority: 3}
  direction: {run_mode: reco, obj_type: particle, optimize: true, priority: 1, radius: 60}
  calo_ke: {run_mode: both, scaling: 1.0, shower_fudge: 1/0.83, priority: 1}
  csda_ke: {run_mode: reco, tracking_mode: step_next, segment_length: 5.0, priority: 1}
  mcs_ke: {run_mode: reco, tracking_mode: bin_pca, segment_length: 5.0}
  containment: {run_mode: both, detector: ndlar_single, margin: 5.0, mode: detector,
    priority: 1}
  vertex: {run_mode: reco, use_primaries: true, update_primaries: false, priority: 1}
  topology_threshold:
    run_mode: both
    ke_thresholds: {4: 50, default: 25}
  fiducial: {run_mode: both, detector: ndlar_single, margin: 15.0, mode: detector}
  children_count: {mode: shape}
  match: {match_mode: both, ghost: false, truth_point_mode: points, fragment: false,
    particle: true, interaction: true}

Will load 1 file(s):
 - /pscratch/sd/d/dunepro/cuddandr/output/FSD_CosmicRun3/run-mlreco/FSD_CosmicRun3.flow2supera/LARCV/0000000/FSD_CosmicRun3.flow2supera.0000069.LARCV.root

Loading tree sparse3d_pcluster
Loading tree sparse3d_pcluster_semantics
Loading tree particle_pcluster
Loading tree cluster3d_pcluster
Loading tree neutrino_mc_truth
Loading tree trigger_base
Loading tree opflash_light

Total number of entries in the file(s): 1989

Total number of entries selected: 1989

Full chain configuration:
  deghosting                 : null
  charge_rescaling           : null
  segmentation               : uresnet
  point_proposal             : ppn
  fragmentation              : graph_spice
  shower_aggregation         : grappa
  shower_primary             : grappa
  track_aggregation          : grappa
  particle_aggregation       : null
  inter_aggregation          : grappa
  particle_identification    : grappa
  primary_identification     : grappa
  orientation_identification : grappa
  calibration                : null

Restoring weights for module full_chain from /dvs_ro/cfs/projectdirs/dune/users/drielsma/weights/ndlar_250505_snapshot.ckpt...
Done.
Iter. 0 (epoch 0.008) @ 2025-08-21 10:32:57
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 8.36 s (83.88 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 1 (epoch 0.016) @ 2025-08-21 10:33:05
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.44 s (36.20 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 2 (epoch 0.024) @ 2025-08-21 10:33:06
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.61 s (23.65 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 3 (epoch 0.032) @ 2025-08-21 10:33:06
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.40 s (38.68 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 4 (epoch 0.040) @ 2025-08-21 10:33:07
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.34 s (40.86 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 5 (epoch 0.048) @ 2025-08-21 10:33:07
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.52 s (27.77 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 6 (epoch 0.056) @ 2025-08-21 10:33:08
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.38 s (38.61 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 7 (epoch 0.064) @ 2025-08-21 10:33:08
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.46 s (46.68 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 8 (epoch 0.072) @ 2025-08-21 10:33:08
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.43 s (52.20 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 9 (epoch 0.080) @ 2025-08-21 10:33:09
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.65 s (62.67 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 10 (epoch 0.088) @ 2025-08-21 10:33:10
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.42 s (41.29 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 11 (epoch 0.096) @ 2025-08-21 10:33:10
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.58 s (29.70 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 12 (epoch 0.104) @ 2025-08-21 10:33:11
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.34 s (41.39 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 13 (epoch 0.112) @ 2025-08-21 10:33:11
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.39 s (33.00 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 14 (epoch 0.120) @ 2025-08-21 10:33:11
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.37 s (39.98 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 15 (epoch 0.128) @ 2025-08-21 10:33:12
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.34 s (40.14 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 16 (epoch 0.136) @ 2025-08-21 10:33:12
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.37 s (37.81 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 17 (epoch 0.144) @ 2025-08-21 10:33:12
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.54 s (47.35 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 18 (epoch 0.152) @ 2025-08-21 10:33:13
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.32 s (40.04 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 19 (epoch 0.160) @ 2025-08-21 10:33:13
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.42 s (46.09 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 20 (epoch 0.168) @ 2025-08-21 10:33:14
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.43 s (42.48 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 21 (epoch 0.176) @ 2025-08-21 10:33:14
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.76 s (62.99 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 22 (epoch 0.184) @ 2025-08-21 10:33:15
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.33 s (39.53 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 23 (epoch 0.192) @ 2025-08-21 10:33:15
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.35 s (47.02 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 24 (epoch 0.200) @ 2025-08-21 10:33:16
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.64 s (58.02 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 25 (epoch 0.208) @ 2025-08-21 10:33:16
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.67 s (26.98 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 26 (epoch 0.216) @ 2025-08-21 10:33:17
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.39 s (38.85 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 27 (epoch 0.224) @ 2025-08-21 10:33:17
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.39 s (49.29 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 28 (epoch 0.232) @ 2025-08-21 10:33:18
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.41 s (38.07 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 29 (epoch 0.240) @ 2025-08-21 10:33:18
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.68 s (52.60 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 30 (epoch 0.248) @ 2025-08-21 10:33:19
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.36 s (39.19 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 31 (epoch 0.256) @ 2025-08-21 10:33:19
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.38 s (38.94 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 32 (epoch 0.264) @ 2025-08-21 10:33:20
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.36 s (38.25 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 33 (epoch 0.272) @ 2025-08-21 10:33:20
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.49 s (42.83 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 34 (epoch 0.280) @ 2025-08-21 10:33:20
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.40 s (38.85 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 35 (epoch 0.288) @ 2025-08-21 10:33:21
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.40 s (33.96 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 36 (epoch 0.296) @ 2025-08-21 10:33:21
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.33 s (39.87 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 37 (epoch 0.304) @ 2025-08-21 10:33:22
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.64 s (34.32 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 38 (epoch 0.312) @ 2025-08-21 10:33:22
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.41 s (37.42 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 39 (epoch 0.320) @ 2025-08-21 10:33:23
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.38 s (41.21 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 40 (epoch 0.328) @ 2025-08-21 10:33:23
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.33 s (41.28 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 41 (epoch 0.336) @ 2025-08-21 10:33:23
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.41 s (41.19 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 42 (epoch 0.344) @ 2025-08-21 10:33:24
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.42 s (39.27 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 43 (epoch 0.352) @ 2025-08-21 10:33:24
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.41 s (35.73 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 44 (epoch 0.360) @ 2025-08-21 10:33:25
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.34 s (40.40 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 45 (epoch 0.368) @ 2025-08-21 10:33:25
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.40 s (33.24 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 46 (epoch 0.376) @ 2025-08-21 10:33:25
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.35 s (38.14 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 47 (epoch 0.384) @ 2025-08-21 10:33:26
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.41 s (39.05 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 48 (epoch 0.392) @ 2025-08-21 10:33:26
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.37 s (40.47 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 49 (epoch 0.400) @ 2025-08-21 10:33:27
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.42 s (35.45 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 50 (epoch 0.408) @ 2025-08-21 10:33:27
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.38 s (38.44 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 51 (epoch 0.416) @ 2025-08-21 10:33:27
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.58 s (30.74 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 52 (epoch 0.424) @ 2025-08-21 10:33:28
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.37 s (40.32 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 53 (epoch 0.432) @ 2025-08-21 10:33:28
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.44 s (32.32 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 54 (epoch 0.440) @ 2025-08-21 10:33:29
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.35 s (47.00 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 55 (epoch 0.448) @ 2025-08-21 10:33:29
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.39 s (39.42 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 56 (epoch 0.456) @ 2025-08-21 10:33:29
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.36 s (39.12 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 57 (epoch 0.464) @ 2025-08-21 10:33:30
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.42 s (35.52 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 58 (epoch 0.472) @ 2025-08-21 10:33:30
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.34 s (39.76 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 59 (epoch 0.480) @ 2025-08-21 10:33:31
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.46 s (39.93 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 60 (epoch 0.488) @ 2025-08-21 10:33:31
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.38 s (40.79 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 61 (epoch 0.496) @ 2025-08-21 10:33:31
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.53 s (42.23 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 62 (epoch 0.504) @ 2025-08-21 10:33:32
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 1.30 s (39.26 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 63 (epoch 0.512) @ 2025-08-21 10:33:33
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.40 s (38.63 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 64 (epoch 0.520) @ 2025-08-21 10:33:34
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.31 s (44.53 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 65 (epoch 0.528) @ 2025-08-21 10:33:34
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.52 s (44.72 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 66 (epoch 0.536) @ 2025-08-21 10:33:35
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.40 s (37.72 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 67 (epoch 0.544) @ 2025-08-21 10:33:35
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.68 s (58.93 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 68 (epoch 0.552) @ 2025-08-21 10:33:36
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.36 s (38.46 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 69 (epoch 0.560) @ 2025-08-21 10:33:36
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.72 s (60.26 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 70 (epoch 0.568) @ 2025-08-21 10:33:37
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.30 s (40.72 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 71 (epoch 0.576) @ 2025-08-21 10:33:37
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.40 s (36.85 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 72 (epoch 0.584) @ 2025-08-21 10:33:37
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.38 s (38.58 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 73 (epoch 0.592) @ 2025-08-21 10:33:38
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.74 s (61.59 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 74 (epoch 0.600) @ 2025-08-21 10:33:39
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.42 s (39.49 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 75 (epoch 0.608) @ 2025-08-21 10:33:39
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.40 s (42.28 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 76 (epoch 0.616) @ 2025-08-21 10:33:39
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.35 s (40.52 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 77 (epoch 0.624) @ 2025-08-21 10:33:40
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.70 s (39.77 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 78 (epoch 0.632) @ 2025-08-21 10:33:40
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.39 s (41.73 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 79 (epoch 0.640) @ 2025-08-21 10:33:41
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.35 s (42.36 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 80 (epoch 0.648) @ 2025-08-21 10:33:41
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.42 s (40.19 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 81 (epoch 0.656) @ 2025-08-21 10:33:42
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.42 s (33.71 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 82 (epoch 0.664) @ 2025-08-21 10:33:42
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.38 s (38.96 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 83 (epoch 0.672) @ 2025-08-21 10:33:42
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.69 s (69.69 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 84 (epoch 0.680) @ 2025-08-21 10:33:43
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.39 s (37.43 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 85 (epoch 0.688) @ 2025-08-21 10:33:44
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.43 s (35.64 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 86 (epoch 0.696) @ 2025-08-21 10:33:44
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.39 s (38.26 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 87 (epoch 0.704) @ 2025-08-21 10:33:44
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.39 s (39.45 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 88 (epoch 0.712) @ 2025-08-21 10:33:45
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.42 s (35.96 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 89 (epoch 0.720) @ 2025-08-21 10:33:45
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.43 s (35.69 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 90 (epoch 0.728) @ 2025-08-21 10:33:46
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.39 s (39.28 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 91 (epoch 0.736) @ 2025-08-21 10:33:46
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.40 s (37.72 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 92 (epoch 0.744) @ 2025-08-21 10:33:46
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.53 s (50.30 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 93 (epoch 0.752) @ 2025-08-21 10:33:47
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.47 s (32.53 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 94 (epoch 0.760) @ 2025-08-21 10:33:47
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.35 s (38.42 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 95 (epoch 0.768) @ 2025-08-21 10:33:48
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.42 s (38.25 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 96 (epoch 0.776) @ 2025-08-21 10:33:48
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.52 s (46.98 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 97 (epoch 0.784) @ 2025-08-21 10:33:49
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.41 s (35.89 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 98 (epoch 0.792) @ 2025-08-21 10:33:49
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.38 s (37.67 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 99 (epoch 0.800) @ 2025-08-21 10:33:50
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.42 s (37.05 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 100 (epoch 0.808) @ 2025-08-21 10:33:50
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.36 s (37.98 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 101 (epoch 0.816) @ 2025-08-21 10:33:50
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.43 s (33.74 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 102 (epoch 0.824) @ 2025-08-21 10:33:51
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.31 s (39.30 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 103 (epoch 0.832) @ 2025-08-21 10:33:51
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.41 s (37.64 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 104 (epoch 0.840) @ 2025-08-21 10:33:51
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.50 s (38.49 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 105 (epoch 0.848) @ 2025-08-21 10:33:52
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.39 s (39.53 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 106 (epoch 0.856) @ 2025-08-21 10:33:52
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.29 s (43.03 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 107 (epoch 0.864) @ 2025-08-21 10:33:53
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.37 s (37.83 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 108 (epoch 0.872) @ 2025-08-21 10:33:53
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.39 s (40.36 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 109 (epoch 0.880) @ 2025-08-21 10:33:53
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.39 s (36.73 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 110 (epoch 0.888) @ 2025-08-21 10:33:54
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.41 s (36.89 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 111 (epoch 0.896) @ 2025-08-21 10:33:54
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.36 s (38.42 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 112 (epoch 0.904) @ 2025-08-21 10:33:55
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.46 s (32.44 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 113 (epoch 0.912) @ 2025-08-21 10:33:55
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.39 s (37.69 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 114 (epoch 0.920) @ 2025-08-21 10:33:55
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.31 s (41.14 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 115 (epoch 0.928) @ 2025-08-21 10:33:56
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.43 s (33.87 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 116 (epoch 0.936) @ 2025-08-21 10:33:56
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.39 s (39.63 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 117 (epoch 0.944) @ 2025-08-21 10:33:57
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.37 s (36.58 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 118 (epoch 0.952) @ 2025-08-21 10:33:57
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.33 s (40.62 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 119 (epoch 0.960) @ 2025-08-21 10:33:57
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.39 s (37.80 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 120 (epoch 0.968) @ 2025-08-21 10:33:58
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.42 s (35.49 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 121 (epoch 0.976) @ 2025-08-21 10:33:58
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.42 s (36.51 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 122 (epoch 0.984) @ 2025-08-21 10:33:59
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.41 s (37.78 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

Iter. 123 (epoch 0.992) @ 2025-08-21 10:33:59
  | Time (inference)    | GPU memory          | Loss     | Accuracy |
  |---------------------+---------------------+----------+----------|
  | 0.59 s (58.39 %)    | 0.23 GB (0.53 %)    | -1.000   | -1.000   |

 *** Break *** segmentation violation



===========================================================
There was a crash.
This is the entire stack trace of all threads:
===========================================================

Thread 59 (Thread 0x7f327efe1640 (LWP 994064) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 58 (Thread 0x7f327f7e2640 (LWP 994063) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 57 (Thread 0x7f327ffe3640 (LWP 994062) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 56 (Thread 0x7f32807e4640 (LWP 994061) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 55 (Thread 0x7f3280fe5640 (LWP 994060) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 54 (Thread 0x7f32817e6640 (LWP 994059) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 53 (Thread 0x7f3281fe7640 (LWP 994058) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 52 (Thread 0x7f32827e8640 (LWP 994057) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 51 (Thread 0x7f3282fe9640 (LWP 994056) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 50 (Thread 0x7f32837ea640 (LWP 994055) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 49 (Thread 0x7f3283feb640 (LWP 994054) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 48 (Thread 0x7f32847ec640 (LWP 994053) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 47 (Thread 0x7f3284fed640 (LWP 994052) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 46 (Thread 0x7f32857ee640 (LWP 994051) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 45 (Thread 0x7f3285fef640 (LWP 994050) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 44 (Thread 0x7f32867f0640 (LWP 994049) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 43 (Thread 0x7f3286ff1640 (LWP 994048) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 42 (Thread 0x7f32877f2640 (LWP 994047) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 41 (Thread 0x7f3287ff3640 (LWP 994046) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 40 (Thread 0x7f32887f4640 (LWP 994045) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 39 (Thread 0x7f3288ff5640 (LWP 994044) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 38 (Thread 0x7f32897f6640 (LWP 994043) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 37 (Thread 0x7f3289ff7640 (LWP 994042) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 36 (Thread 0x7f328a7f8640 (LWP 994041) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 35 (Thread 0x7f328aff9640 (LWP 994040) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 34 (Thread 0x7f328b7fa640 (LWP 994039) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 33 (Thread 0x7f328bffb640 (LWP 994038) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 32 (Thread 0x7f328c7fc640 (LWP 994037) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 31 (Thread 0x7f328cffd640 (LWP 994036) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 30 (Thread 0x7f328d7fe640 (LWP 994035) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 29 (Thread 0x7f328dfff640 (LWP 994034) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 28 (Thread 0x7f32ccffd640 (LWP 993651) "python3"):
#0  __futex_abstimed_wait_common64 (private=<optimized out>, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f32a8001020) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=<optimized out>, abstime=0x0, clockid=0, expected=0, futex_word=0x7f32a8001020) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f32a8001020, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=<optimized out>) at ./nptl/futex-internal.c:139
#3  0x00007f342cd09bdf in do_futex_wait (sem=sem
entry=0x7f32a8001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
#4  0x00007f342cd09c78 in __new_sem_wait_slow64 (sem=0x7f32a8001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
#5  0x000055fd4935ddd0 in PyThread_acquire_lock_timed ()
#6  0x000055fd493b1a64 in ?? ()
#7  0x000055fd4939cde7 in ?? ()
#8  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#9  0x000055fd493a94d1 in ?? ()
#10 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#11 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#12 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#13 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#14 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#17 0x000055fd493a9731 in ?? ()
#18 0x000055fd494be7ca in ?? ()
#19 0x000055fd494b4138 in ?? ()
#20 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#21 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 27 (Thread 0x7f32cd7fe640 (LWP 993650) "python3"):
#0  __futex_abstimed_wait_common64 (private=<optimized out>, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f32b4001020) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=<optimized out>, abstime=0x0, clockid=0, expected=0, futex_word=0x7f32b4001020) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f32b4001020, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=<optimized out>) at ./nptl/futex-internal.c:139
#3  0x00007f342cd09bdf in do_futex_wait (sem=sem
entry=0x7f32b4001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
#4  0x00007f342cd09c78 in __new_sem_wait_slow64 (sem=0x7f32b4001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
#5  0x000055fd4935ddd0 in PyThread_acquire_lock_timed ()
#6  0x000055fd493b1a64 in ?? ()
#7  0x000055fd4939cde7 in ?? ()
#8  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#9  0x000055fd493a94d1 in ?? ()
#10 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#11 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#12 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#13 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#14 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#17 0x000055fd493a9731 in ?? ()
#18 0x000055fd494be7ca in ?? ()
#19 0x000055fd494b4138 in ?? ()
#20 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#21 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 26 (Thread 0x7f32cdfff640 (LWP 993649) "python3"):
#0  __futex_abstimed_wait_common64 (private=<optimized out>, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f32b0001020) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=<optimized out>, abstime=0x0, clockid=0, expected=0, futex_word=0x7f32b0001020) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f32b0001020, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=<optimized out>) at ./nptl/futex-internal.c:139
#3  0x00007f342cd09bdf in do_futex_wait (sem=sem
entry=0x7f32b0001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
#4  0x00007f342cd09c78 in __new_sem_wait_slow64 (sem=0x7f32b0001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
#5  0x000055fd4935ddd0 in PyThread_acquire_lock_timed ()
#6  0x000055fd493b1a64 in ?? ()
#7  0x000055fd4939cde7 in ?? ()
#8  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#9  0x000055fd493a94d1 in ?? ()
#10 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#11 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#12 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#13 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#14 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#17 0x000055fd493a9731 in ?? ()
#18 0x000055fd494be7ca in ?? ()
#19 0x000055fd494b4138 in ?? ()
#20 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#21 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 25 (Thread 0x7f32fcfda640 (LWP 993648) "python3"):
#0  __futex_abstimed_wait_common64 (private=<optimized out>, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f32bc001020) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=<optimized out>, abstime=0x0, clockid=0, expected=0, futex_word=0x7f32bc001020) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f32bc001020, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=<optimized out>) at ./nptl/futex-internal.c:139
#3  0x00007f342cd09bdf in do_futex_wait (sem=sem
entry=0x7f32bc001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
#4  0x00007f342cd09c78 in __new_sem_wait_slow64 (sem=0x7f32bc001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
#5  0x000055fd4935ddd0 in PyThread_acquire_lock_timed ()
#6  0x000055fd493b1a64 in ?? ()
#7  0x000055fd4939cde7 in ?? ()
#8  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#9  0x000055fd493a94d1 in ?? ()
#10 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#11 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#12 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#13 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#14 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#17 0x000055fd493a9731 in ?? ()
#18 0x000055fd494be7ca in ?? ()
#19 0x000055fd494b4138 in ?? ()
#20 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#21 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 24 (Thread 0x7f32fd7db640 (LWP 993647) "python3"):
#0  __futex_abstimed_wait_common64 (private=<optimized out>, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f32b8001020) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=<optimized out>, abstime=0x0, clockid=0, expected=0, futex_word=0x7f32b8001020) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f32b8001020, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=<optimized out>) at ./nptl/futex-internal.c:139
#3  0x00007f342cd09bdf in do_futex_wait (sem=sem
entry=0x7f32b8001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
#4  0x00007f342cd09c78 in __new_sem_wait_slow64 (sem=0x7f32b8001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
#5  0x000055fd4935ddd0 in PyThread_acquire_lock_timed ()
#6  0x000055fd493b1a64 in ?? ()
#7  0x000055fd4939cde7 in ?? ()
#8  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#9  0x000055fd493a94d1 in ?? ()
#10 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#11 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#12 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#13 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#14 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#17 0x000055fd493a9731 in ?? ()
#18 0x000055fd494be7ca in ?? ()
#19 0x000055fd494b4138 in ?? ()
#20 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#21 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 23 (Thread 0x7f32fdfdc640 (LWP 993646) "python3"):
#0  __futex_abstimed_wait_common64 (private=<optimized out>, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f32c4001020) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=<optimized out>, abstime=0x0, clockid=0, expected=0, futex_word=0x7f32c4001020) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f32c4001020, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=<optimized out>) at ./nptl/futex-internal.c:139
#3  0x00007f342cd09bdf in do_futex_wait (sem=sem
entry=0x7f32c4001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
#4  0x00007f342cd09c78 in __new_sem_wait_slow64 (sem=0x7f32c4001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
#5  0x000055fd4935ddd0 in PyThread_acquire_lock_timed ()
#6  0x000055fd493b1a64 in ?? ()
#7  0x000055fd4939cde7 in ?? ()
#8  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#9  0x000055fd493a94d1 in ?? ()
#10 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#11 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#12 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#13 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#14 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#17 0x000055fd493a9731 in ?? ()
#18 0x000055fd494be7ca in ?? ()
#19 0x000055fd494b4138 in ?? ()
#20 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#21 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 22 (Thread 0x7f32fe7dd640 (LWP 993583) "python3"):
#0  __futex_abstimed_wait_common64 (private=<optimized out>, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f32c0001020) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=<optimized out>, abstime=0x0, clockid=0, expected=0, futex_word=0x7f32c0001020) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f32c0001020, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=<optimized out>) at ./nptl/futex-internal.c:139
#3  0x00007f342cd09bdf in do_futex_wait (sem=sem
entry=0x7f32c0001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
#4  0x00007f342cd09c78 in __new_sem_wait_slow64 (sem=0x7f32c0001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
#5  0x000055fd4935ddd0 in PyThread_acquire_lock_timed ()
#6  0x000055fd493b1a64 in ?? ()
#7  0x000055fd4939cde7 in ?? ()
#8  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#9  0x000055fd493a94d1 in ?? ()
#10 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#11 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#12 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#13 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#14 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#17 0x000055fd493a9731 in ?? ()
#18 0x000055fd494be7ca in ?? ()
#19 0x000055fd494b4138 in ?? ()
#20 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#21 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 21 (Thread 0x7f32fefde640 (LWP 993582) "python3"):
#0  __futex_abstimed_wait_common64 (private=<optimized out>, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f32c8001020) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=<optimized out>, abstime=0x0, clockid=0, expected=0, futex_word=0x7f32c8001020) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f32c8001020, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=<optimized out>) at ./nptl/futex-internal.c:139
#3  0x00007f342cd09bdf in do_futex_wait (sem=sem
entry=0x7f32c8001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
#4  0x00007f342cd09c78 in __new_sem_wait_slow64 (sem=0x7f32c8001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
#5  0x000055fd4935ddd0 in PyThread_acquire_lock_timed ()
#6  0x000055fd493b1a64 in ?? ()
#7  0x000055fd4939cde7 in ?? ()
#8  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#9  0x000055fd493a94d1 in ?? ()
#10 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#11 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#12 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#13 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#14 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#17 0x000055fd493a9731 in ?? ()
#18 0x000055fd494be7ca in ?? ()
#19 0x000055fd494b4138 in ?? ()
#20 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#21 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 20 (Thread 0x7f3310ded640 (LWP 993139) "cuda-EvtHandlr"):
#0  0x00007f342cd85bcf in __GI___poll (fds=0x7f32f8000c80, nfds=13, timeout=100) at ../sysdeps/unix/sysv/linux/poll.c:29
#1  0x00007f33718aa44f in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1
#2  0x00007f337196e80f in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1
#3  0x00007f33718a2143 in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1
#4  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#5  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 19 (Thread 0x7f33297e7640 (LWP 993110) "cuda0000340000e"):
#0  0x00007f342cd85bcf in __GI___poll (fds=0x55fd591bd690, nfds=3, timeout=-1) at ../sysdeps/unix/sysv/linux/poll.c:29
#1  0x00007f33718aa44f in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1
#2  0x00007f337196e80f in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1
#3  0x00007f33718a2143 in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1
#4  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#5  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 18 (Thread 0x7f33359ea640 (LWP 993107) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 17 (Thread 0x7f33361eb640 (LWP 993106) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 16 (Thread 0x7f33369ec640 (LWP 993105) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 15 (Thread 0x7f33371ed640 (LWP 993104) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 14 (Thread 0x7f33379ee640 (LWP 993103) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 13 (Thread 0x7f33381ef640 (LWP 993102) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 12 (Thread 0x7f33389f0640 (LWP 993101) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 11 (Thread 0x7f33391f1640 (LWP 993100) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 10 (Thread 0x7f33399f2640 (LWP 993099) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 9 (Thread 0x7f333a1f3640 (LWP 993098) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 8 (Thread 0x7f333a9f4640 (LWP 993097) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 7 (Thread 0x7f333b1f5640 (LWP 993096) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 6 (Thread 0x7f333b9f6640 (LWP 993095) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 5 (Thread 0x7f333c1f7640 (LWP 993094) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 4 (Thread 0x7f333c9f8640 (LWP 993093) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 3 (Thread 0x7f333d1f9640 (LWP 993092) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 2 (Thread 0x7f33489e2640 (LWP 993082) "python3"):
#0  0x00007f342cd8863d in __GI___select (nfds=0, readfds=0x0, writefds=0x0, exceptfds=0x0, timeout=0x7f33489e15d0) at ../sysdeps/unix/sysv/linux/select.c:69
#1  0x000055fd494a62f6 in ?? ()
#2  0x000055fd4939b534 in ?? ()
#3  0x000055fd4938b702 in _PyEval_EvalFrameDefault ()
#4  0x000055fd4939c38c in _PyFunction_Vectorcall ()
#5  0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#6  0x000055fd4939c38c in _PyFunction_Vectorcall ()
#7  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#8  0x000055fd4939c38c in _PyFunction_Vectorcall ()
#9  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#10 0x000055fd493a9731 in ?? ()
#11 0x000055fd494be7ca in ?? ()
#12 0x000055fd494b4138 in ?? ()
#13 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#14 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 1 (Thread 0x7f342cc68740 (LWP 993065) "python3"):
#0  0x00007f342cd5742f in __GI___wait4 (pid=995682, stat_loc=stat_loc
entry=0x7ffff1996de8, options=options
entry=0, usage=usage
entry=0x0) at ../sysdeps/unix/sysv/linux/wait4.c:30
#1  0x00007f342cd573ab in __GI___waitpid (pid=<optimized out>, stat_loc=stat_loc
entry=0x7ffff1996de8, options=options
entry=0) at ./posix/waitpid.c:38
#2  0x00007f342ccbdbdb in do_system (line=<optimized out>) at ../sysdeps/posix/system.c:171
#3  0x00007f335fc7cc84 in TUnixSystem::StackTrace() () from /app/root/lib/libCore.so
#4  0x00007f335ff24ff3 in (anonymous namespace)::TExceptionHandlerImp::HandleException(int) () from /app/root/lib/libcppyy_backend3_10.so
#5  0x00007f335fc79e19 in TUnixSystem::DispatchSignals(ESignals) () from /app/root/lib/libCore.so
#6  <signal handler called>
#7  0x00007f33419ab9f6 in minkowski::CoordinateMapGPU<int, minkowski::detail::c10_allocator>::stride(std::vector<unsigned int, std::allocator<unsigned int> > const&) const () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#8  0x00007f33419e7559 in minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>::stride(std::pair<std::vector<unsigned int, std::allocator<unsigned int> >, std::string> const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::string) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#9  0x00007f334196e07a in at::Tensor minkowski::ConvolutionForwardGPU<int, minkowski::detail::c10_allocator>(at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#10 0x00007f3341ad72d1 in pybind11::cpp_function::initialize<at::Tensor (*&)(at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*), at::Tensor, at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*, pybind11::name, pybind11::scope, pybind11::sibling, pybind11::call_guard<pybind11::gil_scoped_release> >(at::Tensor (*&)(at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*), at::Tensor (*)(at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*), pybind11::name const&, pybind11::scope const&, pybind11::sibling const&, pybind11::call_guard<pybind11::gil_scoped_release> const&)::{lambda(pybind11::detail::function_call&)#3}::_FUN(pybind11::detail::function_call&) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#11 0x00007f33419f2567 in pybind11::cpp_function::dispatcher(_object*, _object*, _object*) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#12 0x000055fd4939bb32 in ?? ()
#13 0x000055fd4939239b in _PyObject_MakeTpCall ()
#14 0x000055fd4938ba97 in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x00007f342b044249 in THPFunction_apply(_object*, _object*) () from /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_python.so
#17 0x000055fd4939bb58 in ?? ()
#18 0x000055fd493aa10b in PyObject_Call ()
#19 0x000055fd4938c6ef in _PyEval_EvalFrameDefault ()
#20 0x000055fd493a94d1 in ?? ()
#21 0x000055fd4938b702 in _PyEval_EvalFrameDefault ()
#22 0x000055fd493a961e in ?? ()
#23 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#24 0x000055fd493a961e in ?? ()
#25 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#26 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#27 0x000055fd493a662c in _PyObject_Call_Prepend ()
#28 0x000055fd494ae464 in ?? ()
#29 0x000055fd4939239b in _PyObject_MakeTpCall ()
#30 0x000055fd4938ba97 in _PyEval_EvalFrameDefault ()
#31 0x000055fd493a961e in ?? ()
#32 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#33 0x000055fd493a961e in ?? ()
#34 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#35 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#36 0x000055fd493a662c in _PyObject_Call_Prepend ()
#37 0x000055fd494ae464 in ?? ()
#38 0x000055fd4939239b in _PyObject_MakeTpCall ()
#39 0x000055fd4938ba97 in _PyEval_EvalFrameDefault ()
#40 0x000055fd493a961e in ?? ()
#41 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#42 0x000055fd493a961e in ?? ()
#43 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#44 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#45 0x000055fd493a662c in _PyObject_Call_Prepend ()
#46 0x000055fd494ae464 in ?? ()
#47 0x000055fd4939239b in _PyObject_MakeTpCall ()
#48 0x000055fd4938c009 in _PyEval_EvalFrameDefault ()
#49 0x000055fd493a961e in ?? ()
#50 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#51 0x000055fd493a961e in ?? ()
#52 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#53 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#54 0x000055fd493a662c in _PyObject_Call_Prepend ()
#55 0x000055fd494ae464 in ?? ()
#56 0x000055fd4939239b in _PyObject_MakeTpCall ()
#57 0x000055fd4938c009 in _PyEval_EvalFrameDefault ()
#58 0x000055fd493a961e in ?? ()
#59 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#60 0x000055fd493a961e in ?? ()
#61 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#62 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#63 0x000055fd493a662c in _PyObject_Call_Prepend ()
#64 0x000055fd494ae464 in ?? ()
#65 0x000055fd4939239b in _PyObject_MakeTpCall ()
#66 0x000055fd4938c009 in _PyEval_EvalFrameDefault ()
#67 0x000055fd493a961e in ?? ()
#68 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#69 0x000055fd493a961e in ?? ()
#70 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#71 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#72 0x000055fd493a662c in _PyObject_Call_Prepend ()
#73 0x000055fd494ae464 in ?? ()
#74 0x000055fd4939239b in _PyObject_MakeTpCall ()
#75 0x000055fd4938c009 in _PyEval_EvalFrameDefault ()
#76 0x000055fd493a94d1 in ?? ()
#77 0x000055fd4938b702 in _PyEval_EvalFrameDefault ()
#78 0x000055fd493a94d1 in ?? ()
#79 0x000055fd493aa172 in PyObject_Call ()
#80 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#81 0x000055fd493a94d1 in ?? ()
#82 0x000055fd493aa172 in PyObject_Call ()
#83 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#84 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#85 0x000055fd4939161d in _PyObject_FastCallDictTstate ()
#86 0x000055fd493a662c in _PyObject_Call_Prepend ()
#87 0x000055fd494ae464 in ?? ()
#88 0x000055fd493aa10b in PyObject_Call ()
#89 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#90 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#91 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#92 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#93 0x000055fd4939161d in _PyObject_FastCallDictTstate ()
#94 0x000055fd493a662c in _PyObject_Call_Prepend ()
#95 0x000055fd494ae464 in ?? ()
#96 0x000055fd4939239b in _PyObject_MakeTpCall ()
#97 0x000055fd4938c99e in _PyEval_EvalFrameDefault ()
#98 0x000055fd493a94d1 in ?? ()
#99 0x000055fd493879ab in _PyEval_EvalFrameDefault ()
#100 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#101 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#102 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#103 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#104 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#105 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#106 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#107 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#108 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#109 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#110 0x000055fd4946ba16 in ?? ()
#111 0x000055fd4946b8e6 in PyEval_EvalCode ()
#112 0x000055fd49492108 in ?? ()
#113 0x000055fd4948c90f in ?? ()
#114 0x000055fd49491ea5 in ?? ()
#115 0x000055fd49491488 in _PyRun_SimpleFileObject ()
#116 0x000055fd49491167 in _PyRun_AnyFileObject ()
#117 0x000055fd494858ae in Py_RunMain ()
#118 0x000055fd4945f93d in Py_BytesMain ()
#119 0x00007f342cc96d90 in __libc_start_call_main (main=main
entry=0x55fd4945f900, argc=argc
entry=10, argv=argv
entry=0x7ffff19a05b8) at ../sysdeps/nptl/libc_start_call_main.h:58
#120 0x00007f342cc96e40 in __libc_start_main_impl (main=0x55fd4945f900, argc=10, argv=0x7ffff19a05b8, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffff19a05a8) at ../csu/libc-start.c:392
#121 0x000055fd4945f835 in _start ()
===========================================================


The lines below might hint at the cause of the crash.
You may get help by asking at the ROOT forum https://root.cern/forum
Only if you are really convinced it is a bug in ROOT then please submit a
report at https://root.cern/bugs Please post the ENTIRE stack trace
from above as an attachment in addition to anything else
that might help us fixing this issue.
===========================================================
#7  0x00007f33419ab9f6 in minkowski::CoordinateMapGPU<int, minkowski::detail::c10_allocator>::stride(std::vector<unsigned int, std::allocator<unsigned int> > const&) const () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#8  0x00007f33419e7559 in minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>::stride(std::pair<std::vector<unsigned int, std::allocator<unsigned int> >, std::string> const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::string) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#9  0x00007f334196e07a in at::Tensor minkowski::ConvolutionForwardGPU<int, minkowski::detail::c10_allocator>(at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#10 0x00007f3341ad72d1 in pybind11::cpp_function::initialize<at::Tensor (*&)(at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*), at::Tensor, at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*, pybind11::name, pybind11::scope, pybind11::sibling, pybind11::call_guard<pybind11::gil_scoped_release> >(at::Tensor (*&)(at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*), at::Tensor (*)(at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*), pybind11::name const&, pybind11::scope const&, pybind11::sibling const&, pybind11::call_guard<pybind11::gil_scoped_release> const&)::{lambda(pybind11::detail::function_call&)#3}::_FUN(pybind11::detail::function_call&) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#11 0x00007f33419f2567 in pybind11::cpp_function::dispatcher(_object*, _object*, _object*) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#12 0x000055fd4939bb32 in ?? ()
#13 0x000055fd4939239b in _PyObject_MakeTpCall ()
#14 0x000055fd4938ba97 in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x00007f342b044249 in THPFunction_apply(_object*, _object*) () from /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_python.so
#17 0x000055fd4939bb58 in ?? ()
#18 0x000055fd493aa10b in PyObject_Call ()
#19 0x000055fd4938c6ef in _PyEval_EvalFrameDefault ()
#20 0x000055fd493a94d1 in ?? ()
#21 0x000055fd4938b702 in _PyEval_EvalFrameDefault ()
#22 0x000055fd493a961e in ?? ()
#23 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#24 0x000055fd493a961e in ?? ()
#25 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#26 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#27 0x000055fd493a662c in _PyObject_Call_Prepend ()
#28 0x000055fd494ae464 in ?? ()
#29 0x000055fd4939239b in _PyObject_MakeTpCall ()
#30 0x000055fd4938ba97 in _PyEval_EvalFrameDefault ()
#31 0x000055fd493a961e in ?? ()
#32 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#33 0x000055fd493a961e in ?? ()
#34 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#35 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#36 0x000055fd493a662c in _PyObject_Call_Prepend ()
#37 0x000055fd494ae464 in ?? ()
#38 0x000055fd4939239b in _PyObject_MakeTpCall ()
#39 0x000055fd4938ba97 in _PyEval_EvalFrameDefault ()
#40 0x000055fd493a961e in ?? ()
#41 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#42 0x000055fd493a961e in ?? ()
#43 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#44 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#45 0x000055fd493a662c in _PyObject_Call_Prepend ()
#46 0x000055fd494ae464 in ?? ()
#47 0x000055fd4939239b in _PyObject_MakeTpCall ()
#48 0x000055fd4938c009 in _PyEval_EvalFrameDefault ()
#49 0x000055fd493a961e in ?? ()
#50 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#51 0x000055fd493a961e in ?? ()
#52 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#53 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#54 0x000055fd493a662c in _PyObject_Call_Prepend ()
#55 0x000055fd494ae464 in ?? ()
#56 0x000055fd4939239b in _PyObject_MakeTpCall ()
#57 0x000055fd4938c009 in _PyEval_EvalFrameDefault ()
#58 0x000055fd493a961e in ?? ()
#59 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#60 0x000055fd493a961e in ?? ()
#61 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#62 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#63 0x000055fd493a662c in _PyObject_Call_Prepend ()
#64 0x000055fd494ae464 in ?? ()
#65 0x000055fd4939239b in _PyObject_MakeTpCall ()
#66 0x000055fd4938c009 in _PyEval_EvalFrameDefault ()
#67 0x000055fd493a961e in ?? ()
#68 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#69 0x000055fd493a961e in ?? ()
#70 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#71 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#72 0x000055fd493a662c in _PyObject_Call_Prepend ()
#73 0x000055fd494ae464 in ?? ()
#74 0x000055fd4939239b in _PyObject_MakeTpCall ()
#75 0x000055fd4938c009 in _PyEval_EvalFrameDefault ()
#76 0x000055fd493a94d1 in ?? ()
#77 0x000055fd4938b702 in _PyEval_EvalFrameDefault ()
#78 0x000055fd493a94d1 in ?? ()
#79 0x000055fd493aa172 in PyObject_Call ()
#80 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#81 0x000055fd493a94d1 in ?? ()
#82 0x000055fd493aa172 in PyObject_Call ()
#83 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#84 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#85 0x000055fd4939161d in _PyObject_FastCallDictTstate ()
#86 0x000055fd493a662c in _PyObject_Call_Prepend ()
#87 0x000055fd494ae464 in ?? ()
#88 0x000055fd493aa10b in PyObject_Call ()
#89 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#90 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#91 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#92 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#93 0x000055fd4939161d in _PyObject_FastCallDictTstate ()
#94 0x000055fd493a662c in _PyObject_Call_Prepend ()
#95 0x000055fd494ae464 in ?? ()
#96 0x000055fd4939239b in _PyObject_MakeTpCall ()
#97 0x000055fd4938c99e in _PyEval_EvalFrameDefault ()
#98 0x000055fd493a94d1 in ?? ()
#99 0x000055fd493879ab in _PyEval_EvalFrameDefault ()
#100 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#101 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#102 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#103 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#104 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#105 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#106 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#107 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#108 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#109 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#110 0x000055fd4946ba16 in ?? ()
#111 0x000055fd4946b8e6 in PyEval_EvalCode ()
#112 0x000055fd49492108 in ?? ()
#113 0x000055fd4948c90f in ?? ()
#114 0x000055fd49491ea5 in ?? ()
#115 0x000055fd49491488 in _PyRun_SimpleFileObject ()
#116 0x000055fd49491167 in _PyRun_AnyFileObject ()
#117 0x000055fd494858ae in Py_RunMain ()
#118 0x000055fd4945f93d in Py_BytesMain ()
#119 0x00007f342cc96d90 in __libc_start_call_main (main=main
entry=0x55fd4945f900, argc=argc
entry=10, argv=argv
entry=0x7ffff19a05b8) at ../sysdeps/nptl/libc_start_call_main.h:58
#120 0x00007f342cc96e40 in __libc_start_main_impl (main=0x55fd4945f900, argc=10, argv=0x7ffff19a05b8, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffff19a05a8) at ../csu/libc-start.c:392
#121 0x000055fd4945f835 in _start ()
===========================================================


 *** Break *** segmentation violation



===========================================================
There was a crash.
This is the entire stack trace of all threads:
===========================================================

Thread 59 (Thread 0x7f327efe1640 (LWP 994064) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 58 (Thread 0x7f327f7e2640 (LWP 994063) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 57 (Thread 0x7f327ffe3640 (LWP 994062) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 56 (Thread 0x7f32807e4640 (LWP 994061) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 55 (Thread 0x7f3280fe5640 (LWP 994060) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 54 (Thread 0x7f32817e6640 (LWP 994059) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 53 (Thread 0x7f3281fe7640 (LWP 994058) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 52 (Thread 0x7f32827e8640 (LWP 994057) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 51 (Thread 0x7f3282fe9640 (LWP 994056) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 50 (Thread 0x7f32837ea640 (LWP 994055) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 49 (Thread 0x7f3283feb640 (LWP 994054) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 48 (Thread 0x7f32847ec640 (LWP 994053) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 47 (Thread 0x7f3284fed640 (LWP 994052) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 46 (Thread 0x7f32857ee640 (LWP 994051) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 45 (Thread 0x7f3285fef640 (LWP 994050) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 44 (Thread 0x7f32867f0640 (LWP 994049) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 43 (Thread 0x7f3286ff1640 (LWP 994048) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 42 (Thread 0x7f32877f2640 (LWP 994047) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 41 (Thread 0x7f3287ff3640 (LWP 994046) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 40 (Thread 0x7f32887f4640 (LWP 994045) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 39 (Thread 0x7f3288ff5640 (LWP 994044) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 38 (Thread 0x7f32897f6640 (LWP 994043) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 37 (Thread 0x7f3289ff7640 (LWP 994042) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 36 (Thread 0x7f328a7f8640 (LWP 994041) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 35 (Thread 0x7f328aff9640 (LWP 994040) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 34 (Thread 0x7f328b7fa640 (LWP 994039) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 33 (Thread 0x7f328bffb640 (LWP 994038) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 32 (Thread 0x7f328c7fc640 (LWP 994037) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 31 (Thread 0x7f328cffd640 (LWP 994036) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 30 (Thread 0x7f328d7fe640 (LWP 994035) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 29 (Thread 0x7f328dfff640 (LWP 994034) "python3"):
#0  0x00007f342bc18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#1  0x00007f342bc163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1
#2  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#3  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 28 (Thread 0x7f32ccffd640 (LWP 993651) "python3"):
#0  __futex_abstimed_wait_common64 (private=<optimized out>, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f32a8001020) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=<optimized out>, abstime=0x0, clockid=0, expected=0, futex_word=0x7f32a8001020) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f32a8001020, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=<optimized out>) at ./nptl/futex-internal.c:139
#3  0x00007f342cd09bdf in do_futex_wait (sem=sem
entry=0x7f32a8001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
#4  0x00007f342cd09c78 in __new_sem_wait_slow64 (sem=0x7f32a8001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
#5  0x000055fd4935ddd0 in PyThread_acquire_lock_timed ()
#6  0x000055fd493b1a64 in ?? ()
#7  0x000055fd4939cde7 in ?? ()
#8  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#9  0x000055fd493a94d1 in ?? ()
#10 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#11 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#12 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#13 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#14 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#17 0x000055fd493a9731 in ?? ()
#18 0x000055fd494be7ca in ?? ()
#19 0x000055fd494b4138 in ?? ()
#20 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#21 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 27 (Thread 0x7f32cd7fe640 (LWP 993650) "python3"):
#0  __futex_abstimed_wait_common64 (private=<optimized out>, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f32b4001020) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=<optimized out>, abstime=0x0, clockid=0, expected=0, futex_word=0x7f32b4001020) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f32b4001020, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=<optimized out>) at ./nptl/futex-internal.c:139
#3  0x00007f342cd09bdf in do_futex_wait (sem=sem
entry=0x7f32b4001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
#4  0x00007f342cd09c78 in __new_sem_wait_slow64 (sem=0x7f32b4001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
#5  0x000055fd4935ddd0 in PyThread_acquire_lock_timed ()
#6  0x000055fd493b1a64 in ?? ()
#7  0x000055fd4939cde7 in ?? ()
#8  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#9  0x000055fd493a94d1 in ?? ()
#10 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#11 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#12 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#13 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#14 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#17 0x000055fd493a9731 in ?? ()
#18 0x000055fd494be7ca in ?? ()
#19 0x000055fd494b4138 in ?? ()
#20 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#21 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 26 (Thread 0x7f32cdfff640 (LWP 993649) "python3"):
#0  __futex_abstimed_wait_common64 (private=<optimized out>, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f32b0001020) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=<optimized out>, abstime=0x0, clockid=0, expected=0, futex_word=0x7f32b0001020) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f32b0001020, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=<optimized out>) at ./nptl/futex-internal.c:139
#3  0x00007f342cd09bdf in do_futex_wait (sem=sem
entry=0x7f32b0001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
#4  0x00007f342cd09c78 in __new_sem_wait_slow64 (sem=0x7f32b0001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
#5  0x000055fd4935ddd0 in PyThread_acquire_lock_timed ()
#6  0x000055fd493b1a64 in ?? ()
#7  0x000055fd4939cde7 in ?? ()
#8  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#9  0x000055fd493a94d1 in ?? ()
#10 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#11 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#12 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#13 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#14 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#17 0x000055fd493a9731 in ?? ()
#18 0x000055fd494be7ca in ?? ()
#19 0x000055fd494b4138 in ?? ()
#20 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#21 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 25 (Thread 0x7f32fcfda640 (LWP 993648) "python3"):
#0  __futex_abstimed_wait_common64 (private=<optimized out>, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f32bc001020) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=<optimized out>, abstime=0x0, clockid=0, expected=0, futex_word=0x7f32bc001020) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f32bc001020, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=<optimized out>) at ./nptl/futex-internal.c:139
#3  0x00007f342cd09bdf in do_futex_wait (sem=sem
entry=0x7f32bc001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
#4  0x00007f342cd09c78 in __new_sem_wait_slow64 (sem=0x7f32bc001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
#5  0x000055fd4935ddd0 in PyThread_acquire_lock_timed ()
#6  0x000055fd493b1a64 in ?? ()
#7  0x000055fd4939cde7 in ?? ()
#8  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#9  0x000055fd493a94d1 in ?? ()
#10 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#11 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#12 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#13 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#14 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#17 0x000055fd493a9731 in ?? ()
#18 0x000055fd494be7ca in ?? ()
#19 0x000055fd494b4138 in ?? ()
#20 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#21 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 24 (Thread 0x7f32fd7db640 (LWP 993647) "python3"):
#0  __futex_abstimed_wait_common64 (private=<optimized out>, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f32b8001020) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=<optimized out>, abstime=0x0, clockid=0, expected=0, futex_word=0x7f32b8001020) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f32b8001020, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=<optimized out>) at ./nptl/futex-internal.c:139
#3  0x00007f342cd09bdf in do_futex_wait (sem=sem
entry=0x7f32b8001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
#4  0x00007f342cd09c78 in __new_sem_wait_slow64 (sem=0x7f32b8001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
#5  0x000055fd4935ddd0 in PyThread_acquire_lock_timed ()
#6  0x000055fd493b1a64 in ?? ()
#7  0x000055fd4939cde7 in ?? ()
#8  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#9  0x000055fd493a94d1 in ?? ()
#10 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#11 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#12 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#13 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#14 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#17 0x000055fd493a9731 in ?? ()
#18 0x000055fd494be7ca in ?? ()
#19 0x000055fd494b4138 in ?? ()
#20 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#21 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 23 (Thread 0x7f32fdfdc640 (LWP 993646) "python3"):
#0  __futex_abstimed_wait_common64 (private=<optimized out>, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f32c4001020) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=<optimized out>, abstime=0x0, clockid=0, expected=0, futex_word=0x7f32c4001020) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f32c4001020, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=<optimized out>) at ./nptl/futex-internal.c:139
#3  0x00007f342cd09bdf in do_futex_wait (sem=sem
entry=0x7f32c4001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
#4  0x00007f342cd09c78 in __new_sem_wait_slow64 (sem=0x7f32c4001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
#5  0x000055fd4935ddd0 in PyThread_acquire_lock_timed ()
#6  0x000055fd493b1a64 in ?? ()
#7  0x000055fd4939cde7 in ?? ()
#8  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#9  0x000055fd493a94d1 in ?? ()
#10 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#11 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#12 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#13 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#14 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#17 0x000055fd493a9731 in ?? ()
#18 0x000055fd494be7ca in ?? ()
#19 0x000055fd494b4138 in ?? ()
#20 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#21 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 22 (Thread 0x7f32fe7dd640 (LWP 993583) "python3"):
#0  __futex_abstimed_wait_common64 (private=<optimized out>, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f32c0001020) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=<optimized out>, abstime=0x0, clockid=0, expected=0, futex_word=0x7f32c0001020) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f32c0001020, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=<optimized out>) at ./nptl/futex-internal.c:139
#3  0x00007f342cd09bdf in do_futex_wait (sem=sem
entry=0x7f32c0001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
#4  0x00007f342cd09c78 in __new_sem_wait_slow64 (sem=0x7f32c0001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
#5  0x000055fd4935ddd0 in PyThread_acquire_lock_timed ()
#6  0x000055fd493b1a64 in ?? ()
#7  0x000055fd4939cde7 in ?? ()
#8  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#9  0x000055fd493a94d1 in ?? ()
#10 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#11 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#12 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#13 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#14 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#17 0x000055fd493a9731 in ?? ()
#18 0x000055fd494be7ca in ?? ()
#19 0x000055fd494b4138 in ?? ()
#20 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#21 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 21 (Thread 0x7f32fefde640 (LWP 993582) "python3"):
#0  __futex_abstimed_wait_common64 (private=<optimized out>, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f32c8001020) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=<optimized out>, abstime=0x0, clockid=0, expected=0, futex_word=0x7f32c8001020) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f32c8001020, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=<optimized out>) at ./nptl/futex-internal.c:139
#3  0x00007f342cd09bdf in do_futex_wait (sem=sem
entry=0x7f32c8001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
#4  0x00007f342cd09c78 in __new_sem_wait_slow64 (sem=0x7f32c8001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
#5  0x000055fd4935ddd0 in PyThread_acquire_lock_timed ()
#6  0x000055fd493b1a64 in ?? ()
#7  0x000055fd4939cde7 in ?? ()
#8  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#9  0x000055fd493a94d1 in ?? ()
#10 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#11 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#12 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#13 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#14 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#17 0x000055fd493a9731 in ?? ()
#18 0x000055fd494be7ca in ?? ()
#19 0x000055fd494b4138 in ?? ()
#20 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#21 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 20 (Thread 0x7f3310ded640 (LWP 993139) "cuda-EvtHandlr"):
#0  0x00007f342cd85bcf in __GI___poll (fds=0x7f32f8000c80, nfds=13, timeout=100) at ../sysdeps/unix/sysv/linux/poll.c:29
#1  0x00007f33718aa44f in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1
#2  0x00007f337196e80f in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1
#3  0x00007f33718a2143 in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1
#4  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#5  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 19 (Thread 0x7f33297e7640 (LWP 993110) "cuda0000340000e"):
#0  0x00007f342cd85bcf in __GI___poll (fds=0x55fd591bd690, nfds=3, timeout=-1) at ../sysdeps/unix/sysv/linux/poll.c:29
#1  0x00007f33718aa44f in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1
#2  0x00007f337196e80f in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1
#3  0x00007f33718a2143 in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1
#4  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#5  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 18 (Thread 0x7f33359ea640 (LWP 993107) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 17 (Thread 0x7f33361eb640 (LWP 993106) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 16 (Thread 0x7f33369ec640 (LWP 993105) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 15 (Thread 0x7f33371ed640 (LWP 993104) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 14 (Thread 0x7f33379ee640 (LWP 993103) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 13 (Thread 0x7f33381ef640 (LWP 993102) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 12 (Thread 0x7f33389f0640 (LWP 993101) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 11 (Thread 0x7f33391f1640 (LWP 993100) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 10 (Thread 0x7f33399f2640 (LWP 993099) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 9 (Thread 0x7f333a1f3640 (LWP 993098) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 8 (Thread 0x7f333a9f4640 (LWP 993097) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 7 (Thread 0x7f333b1f5640 (LWP 993096) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 6 (Thread 0x7f333b9f6640 (LWP 993095) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 5 (Thread 0x7f333c1f7640 (LWP 993094) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 4 (Thread 0x7f333c9f8640 (LWP 993093) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 3 (Thread 0x7f333d1f9640 (LWP 993092) "python3"):
#0  __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:57
#1  __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f333d227bb8 <gs+184>) at ./nptl/futex-internal.c:87
#2  __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word
entry=0x7f333d227bb8 <gs+184>, expected=expected
entry=0, clockid=clockid
entry=0, abstime=abstime
entry=0x0, private=private
entry=0) at ./nptl/futex-internal.c:139
#3  0x00007f342cd00a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f333d227b68 <gs+104>, cond=0x7f333d227b90 <gs+144>) at ./nptl/pthread_cond_wait.c:503
#4  ___pthread_cond_wait (cond=0x7f333d227b90 <gs+144>, mutex=0x7f333d227b68 <gs+104>) at ./nptl/pthread_cond_wait.c:627
#5  0x00007f333d221cef in th_worker (tidptr=<optimized out>) at numexpr/module.cpp:62
#6  0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#7  0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 2 (Thread 0x7f33489e2640 (LWP 993082) "python3"):
#0  0x00007f342cd8863d in __GI___select (nfds=0, readfds=0x0, writefds=0x0, exceptfds=0x0, timeout=0x7f33489e15d0) at ../sysdeps/unix/sysv/linux/select.c:69
#1  0x000055fd494a62f6 in ?? ()
#2  0x000055fd4939b534 in ?? ()
#3  0x000055fd4938b702 in _PyEval_EvalFrameDefault ()
#4  0x000055fd4939c38c in _PyFunction_Vectorcall ()
#5  0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#6  0x000055fd4939c38c in _PyFunction_Vectorcall ()
#7  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#8  0x000055fd4939c38c in _PyFunction_Vectorcall ()
#9  0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#10 0x000055fd493a9731 in ?? ()
#11 0x000055fd494be7ca in ?? ()
#12 0x000055fd494b4138 in ?? ()
#13 0x00007f342cd01ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#14 0x00007f342cd93850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

Thread 1 (Thread 0x7f342cc68740 (LWP 993065) "python3"):
#0  0x00007f342cd5742f in __GI___wait4 (pid=998447, stat_loc=stat_loc
entry=0x7ffff1996de8, options=options
entry=0, usage=usage
entry=0x0) at ../sysdeps/unix/sysv/linux/wait4.c:30
#1  0x00007f342cd573ab in __GI___waitpid (pid=<optimized out>, stat_loc=stat_loc
entry=0x7ffff1996de8, options=options
entry=0) at ./posix/waitpid.c:38
#2  0x00007f342ccbdbdb in do_system (line=<optimized out>) at ../sysdeps/posix/system.c:171
#3  0x00007f335fc7cc84 in TUnixSystem::StackTrace() () from /app/root/lib/libCore.so
#4  0x00007f335ff24e77 in (anonymous namespace)::TExceptionHandlerImp::HandleException(int) () from /app/root/lib/libcppyy_backend3_10.so
#5  0x00007f335fc79e19 in TUnixSystem::DispatchSignals(ESignals) () from /app/root/lib/libCore.so
#6  <signal handler called>
#7  0x00007f33419ab9f6 in minkowski::CoordinateMapGPU<int, minkowski::detail::c10_allocator>::stride(std::vector<unsigned int, std::allocator<unsigned int> > const&) const () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#8  0x00007f33419e7559 in minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>::stride(std::pair<std::vector<unsigned int, std::allocator<unsigned int> >, std::string> const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::string) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#9  0x00007f334196e07a in at::Tensor minkowski::ConvolutionForwardGPU<int, minkowski::detail::c10_allocator>(at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#10 0x00007f3341ad72d1 in pybind11::cpp_function::initialize<at::Tensor (*&)(at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*), at::Tensor, at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*, pybind11::name, pybind11::scope, pybind11::sibling, pybind11::call_guard<pybind11::gil_scoped_release> >(at::Tensor (*&)(at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*), at::Tensor (*)(at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*), pybind11::name const&, pybind11::scope const&, pybind11::sibling const&, pybind11::call_guard<pybind11::gil_scoped_release> const&)::{lambda(pybind11::detail::function_call&)#3}::_FUN(pybind11::detail::function_call&) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#11 0x00007f33419f2567 in pybind11::cpp_function::dispatcher(_object*, _object*, _object*) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#12 0x000055fd4939bb32 in ?? ()
#13 0x000055fd4939239b in _PyObject_MakeTpCall ()
#14 0x000055fd4938ba97 in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x00007f342b044249 in THPFunction_apply(_object*, _object*) () from /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_python.so
#17 0x000055fd4939bb58 in ?? ()
#18 0x000055fd493aa10b in PyObject_Call ()
#19 0x000055fd4938c6ef in _PyEval_EvalFrameDefault ()
#20 0x000055fd493a94d1 in ?? ()
#21 0x000055fd4938b702 in _PyEval_EvalFrameDefault ()
#22 0x000055fd493a961e in ?? ()
#23 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#24 0x000055fd493a961e in ?? ()
#25 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#26 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#27 0x000055fd493a662c in _PyObject_Call_Prepend ()
#28 0x000055fd494ae464 in ?? ()
#29 0x000055fd4939239b in _PyObject_MakeTpCall ()
#30 0x000055fd4938ba97 in _PyEval_EvalFrameDefault ()
#31 0x000055fd493a961e in ?? ()
#32 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#33 0x000055fd493a961e in ?? ()
#34 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#35 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#36 0x000055fd493a662c in _PyObject_Call_Prepend ()
#37 0x000055fd494ae464 in ?? ()
#38 0x000055fd4939239b in _PyObject_MakeTpCall ()
#39 0x000055fd4938ba97 in _PyEval_EvalFrameDefault ()
#40 0x000055fd493a961e in ?? ()
#41 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#42 0x000055fd493a961e in ?? ()
#43 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#44 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#45 0x000055fd493a662c in _PyObject_Call_Prepend ()
#46 0x000055fd494ae464 in ?? ()
#47 0x000055fd4939239b in _PyObject_MakeTpCall ()
#48 0x000055fd4938c009 in _PyEval_EvalFrameDefault ()
#49 0x000055fd493a961e in ?? ()
#50 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#51 0x000055fd493a961e in ?? ()
#52 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#53 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#54 0x000055fd493a662c in _PyObject_Call_Prepend ()
#55 0x000055fd494ae464 in ?? ()
#56 0x000055fd4939239b in _PyObject_MakeTpCall ()
#57 0x000055fd4938c009 in _PyEval_EvalFrameDefault ()
#58 0x000055fd493a961e in ?? ()
#59 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#60 0x000055fd493a961e in ?? ()
#61 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#62 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#63 0x000055fd493a662c in _PyObject_Call_Prepend ()
#64 0x000055fd494ae464 in ?? ()
#65 0x000055fd4939239b in _PyObject_MakeTpCall ()
#66 0x000055fd4938c009 in _PyEval_EvalFrameDefault ()
#67 0x000055fd493a961e in ?? ()
#68 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#69 0x000055fd493a961e in ?? ()
#70 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#71 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#72 0x000055fd493a662c in _PyObject_Call_Prepend ()
#73 0x000055fd494ae464 in ?? ()
#74 0x000055fd4939239b in _PyObject_MakeTpCall ()
#75 0x000055fd4938c009 in _PyEval_EvalFrameDefault ()
#76 0x000055fd493a94d1 in ?? ()
#77 0x000055fd4938b702 in _PyEval_EvalFrameDefault ()
#78 0x000055fd493a94d1 in ?? ()
#79 0x000055fd493aa172 in PyObject_Call ()
#80 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#81 0x000055fd493a94d1 in ?? ()
#82 0x000055fd493aa172 in PyObject_Call ()
#83 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#84 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#85 0x000055fd4939161d in _PyObject_FastCallDictTstate ()
#86 0x000055fd493a662c in _PyObject_Call_Prepend ()
#87 0x000055fd494ae464 in ?? ()
#88 0x000055fd493aa10b in PyObject_Call ()
#89 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#90 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#91 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#92 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#93 0x000055fd4939161d in _PyObject_FastCallDictTstate ()
#94 0x000055fd493a662c in _PyObject_Call_Prepend ()
#95 0x000055fd494ae464 in ?? ()
#96 0x000055fd4939239b in _PyObject_MakeTpCall ()
#97 0x000055fd4938c99e in _PyEval_EvalFrameDefault ()
#98 0x000055fd493a94d1 in ?? ()
#99 0x000055fd493879ab in _PyEval_EvalFrameDefault ()
#100 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#101 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#102 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#103 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#104 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#105 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#106 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#107 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#108 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#109 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#110 0x000055fd4946ba16 in ?? ()
#111 0x000055fd4946b8e6 in PyEval_EvalCode ()
#112 0x000055fd49492108 in ?? ()
#113 0x000055fd4948c90f in ?? ()
#114 0x000055fd49491ea5 in ?? ()
#115 0x000055fd49491488 in _PyRun_SimpleFileObject ()
#116 0x000055fd49491167 in _PyRun_AnyFileObject ()
#117 0x000055fd494858ae in Py_RunMain ()
#118 0x000055fd4945f93d in Py_BytesMain ()
#119 0x00007f342cc96d90 in __libc_start_call_main (main=main
entry=0x55fd4945f900, argc=argc
entry=10, argv=argv
entry=0x7ffff19a05b8) at ../sysdeps/nptl/libc_start_call_main.h:58
#120 0x00007f342cc96e40 in __libc_start_main_impl (main=0x55fd4945f900, argc=10, argv=0x7ffff19a05b8, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffff19a05a8) at ../csu/libc-start.c:392
#121 0x000055fd4945f835 in _start ()
===========================================================


The lines below might hint at the cause of the crash.
You may get help by asking at the ROOT forum https://root.cern/forum
Only if you are really convinced it is a bug in ROOT then please submit a
report at https://root.cern/bugs Please post the ENTIRE stack trace
from above as an attachment in addition to anything else
that might help us fixing this issue.
===========================================================
#7  0x00007f33419ab9f6 in minkowski::CoordinateMapGPU<int, minkowski::detail::c10_allocator>::stride(std::vector<unsigned int, std::allocator<unsigned int> > const&) const () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#8  0x00007f33419e7559 in minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>::stride(std::pair<std::vector<unsigned int, std::allocator<unsigned int> >, std::string> const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::string) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#9  0x00007f334196e07a in at::Tensor minkowski::ConvolutionForwardGPU<int, minkowski::detail::c10_allocator>(at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#10 0x00007f3341ad72d1 in pybind11::cpp_function::initialize<at::Tensor (*&)(at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*), at::Tensor, at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*, pybind11::name, pybind11::scope, pybind11::sibling, pybind11::call_guard<pybind11::gil_scoped_release> >(at::Tensor (*&)(at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*), at::Tensor (*)(at::Tensor const&, at::Tensor const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<unsigned int, std::allocator<unsigned int> > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager<int, float, minkowski::detail::c10_allocator, minkowski::CoordinateMapGPU>*), pybind11::name const&, pybind11::scope const&, pybind11::sibling const&, pybind11::call_guard<pybind11::gil_scoped_release> const&)::{lambda(pybind11::detail::function_call&)#3}::_FUN(pybind11::detail::function_call&) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#11 0x00007f33419f2567 in pybind11::cpp_function::dispatcher(_object*, _object*, _object*) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so
#12 0x000055fd4939bb32 in ?? ()
#13 0x000055fd4939239b in _PyObject_MakeTpCall ()
#14 0x000055fd4938ba97 in _PyEval_EvalFrameDefault ()
#15 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#16 0x00007f342b044249 in THPFunction_apply(_object*, _object*) () from /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_python.so
#17 0x000055fd4939bb58 in ?? ()
#18 0x000055fd493aa10b in PyObject_Call ()
#19 0x000055fd4938c6ef in _PyEval_EvalFrameDefault ()
#20 0x000055fd493a94d1 in ?? ()
#21 0x000055fd4938b702 in _PyEval_EvalFrameDefault ()
#22 0x000055fd493a961e in ?? ()
#23 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#24 0x000055fd493a961e in ?? ()
#25 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#26 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#27 0x000055fd493a662c in _PyObject_Call_Prepend ()
#28 0x000055fd494ae464 in ?? ()
#29 0x000055fd4939239b in _PyObject_MakeTpCall ()
#30 0x000055fd4938ba97 in _PyEval_EvalFrameDefault ()
#31 0x000055fd493a961e in ?? ()
#32 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#33 0x000055fd493a961e in ?? ()
#34 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#35 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#36 0x000055fd493a662c in _PyObject_Call_Prepend ()
#37 0x000055fd494ae464 in ?? ()
#38 0x000055fd4939239b in _PyObject_MakeTpCall ()
#39 0x000055fd4938ba97 in _PyEval_EvalFrameDefault ()
#40 0x000055fd493a961e in ?? ()
#41 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#42 0x000055fd493a961e in ?? ()
#43 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#44 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#45 0x000055fd493a662c in _PyObject_Call_Prepend ()
#46 0x000055fd494ae464 in ?? ()
#47 0x000055fd4939239b in _PyObject_MakeTpCall ()
#48 0x000055fd4938c009 in _PyEval_EvalFrameDefault ()
#49 0x000055fd493a961e in ?? ()
#50 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#51 0x000055fd493a961e in ?? ()
#52 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#53 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#54 0x000055fd493a662c in _PyObject_Call_Prepend ()
#55 0x000055fd494ae464 in ?? ()
#56 0x000055fd4939239b in _PyObject_MakeTpCall ()
#57 0x000055fd4938c009 in _PyEval_EvalFrameDefault ()
#58 0x000055fd493a961e in ?? ()
#59 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#60 0x000055fd493a961e in ?? ()
#61 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#62 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#63 0x000055fd493a662c in _PyObject_Call_Prepend ()
#64 0x000055fd494ae464 in ?? ()
#65 0x000055fd4939239b in _PyObject_MakeTpCall ()
#66 0x000055fd4938c009 in _PyEval_EvalFrameDefault ()
#67 0x000055fd493a961e in ?? ()
#68 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#69 0x000055fd493a961e in ?? ()
#70 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#71 0x000055fd49391574 in _PyObject_FastCallDictTstate ()
#72 0x000055fd493a662c in _PyObject_Call_Prepend ()
#73 0x000055fd494ae464 in ?? ()
#74 0x000055fd4939239b in _PyObject_MakeTpCall ()
#75 0x000055fd4938c009 in _PyEval_EvalFrameDefault ()
#76 0x000055fd493a94d1 in ?? ()
#77 0x000055fd4938b702 in _PyEval_EvalFrameDefault ()
#78 0x000055fd493a94d1 in ?? ()
#79 0x000055fd493aa172 in PyObject_Call ()
#80 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#81 0x000055fd493a94d1 in ?? ()
#82 0x000055fd493aa172 in PyObject_Call ()
#83 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#84 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#85 0x000055fd4939161d in _PyObject_FastCallDictTstate ()
#86 0x000055fd493a662c in _PyObject_Call_Prepend ()
#87 0x000055fd494ae464 in ?? ()
#88 0x000055fd493aa10b in PyObject_Call ()
#89 0x000055fd49388c30 in _PyEval_EvalFrameDefault ()
#90 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#91 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#92 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#93 0x000055fd4939161d in _PyObject_FastCallDictTstate ()
#94 0x000055fd493a662c in _PyObject_Call_Prepend ()
#95 0x000055fd494ae464 in ?? ()
#96 0x000055fd4939239b in _PyObject_MakeTpCall ()
#97 0x000055fd4938c99e in _PyEval_EvalFrameDefault ()
#98 0x000055fd493a94d1 in ?? ()
#99 0x000055fd493879ab in _PyEval_EvalFrameDefault ()
#100 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#101 0x000055fd4938697f in _PyEval_EvalFrameDefault ()
#102 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#103 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#104 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#105 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#106 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#107 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#108 0x000055fd4939c38c in _PyFunction_Vectorcall ()
#109 0x000055fd49386790 in _PyEval_EvalFrameDefault ()
#110 0x000055fd4946ba16 in ?? ()
#111 0x000055fd4946b8e6 in PyEval_EvalCode ()
#112 0x000055fd49492108 in ?? ()
#113 0x000055fd4948c90f in ?? ()
#114 0x000055fd49491ea5 in ?? ()
#115 0x000055fd49491488 in _PyRun_SimpleFileObject ()
#116 0x000055fd49491167 in _PyRun_AnyFileObject ()
#117 0x000055fd494858ae in Py_RunMain ()
#118 0x000055fd4945f93d in Py_BytesMain ()
#119 0x00007f342cc96d90 in __libc_start_call_main (main=main
entry=0x55fd4945f900, argc=argc
entry=10, argv=argv
entry=0x7ffff19a05b8) at ../sysdeps/nptl/libc_start_call_main.h:58
#120 0x00007f342cc96e40 in __libc_start_main_impl (main=0x55fd4945f900, argc=10, argv=0x7ffff19a05b8, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffff19a05a8) at ../csu/libc-start.c:392
#121 0x000055fd4945f835 in _start ()
===========================================================


Exception in thread Thread-1 (_process_root_events):
Traceback (most recent call last):
  File "/usr/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
    self.run()
  File "/usr/lib/python3.10/threading.py", line 953, in run
    self._target(*self._args, **self._kwargs)
  File "/app/root/lib/ROOT/_application.py", line 91, in _process_root_events
    gSystem.ProcessEvents()
ReferenceError: attempt to access a null-pointer