RUNNING python3 install/spine/bin/run.py --config /tmp/tmp.T7RXg6yei8/ndlar_single_full_chain_250505.cfg --log_dir /pscratch/sd/d/dunepro/cuddandr/logs/FSD_CosmicRun3/run-mlreco/FSD_CosmicRun3.spine/LOGS/0000000 --source /pscratch/sd/d/dunepro/cuddandr/output/FSD_CosmicRun3/run-mlreco/FSD_CosmicRun3.flow2supera/LARCV/0000000/FSD_CosmicRun3.flow2supera.0000177.LARCV.root --output /pscratch/sd/d/dunepro/cuddandr/output/FSD_CosmicRun3/tmp/run-mlreco/FSD_CosmicRun3.spine/FSD_CosmicRun3.spine.0000177.MLRECO_SPINE.hdf5 ██████████ ██████████ ███ ███ ██ ███████████ ███ █ ██ ███ █ █████ ██ ██ ████████ ██ ███ ███ ██ ████ ██ ██████████ █ ███ ██████████ █ ██ █████ ██ ██████████ ██ ███ ██ ███ ███████████ Release version: 0.5.1 $CUDA_VISIBLE_DEVICES=0 Configuration processed at: Linux nid003520 5.14.21-150500.55.97_13.0.78-cray_shasta_c #1 SMP Thu Mar 13 20:09:44 UTC 2025 (330b47d) x86_64 x86_64 x86_64 GNU/Linux base: {world_size: 1, iterations: -1, seed: 0, dtype: float32, unwrap: true, log_dir: /pscratch/sd/d/dunepro/cuddandr/logs/FSD_CosmicRun3/run-mlreco/FSD_CosmicRun3.spine/LOGS/0000000, prefix_log: true, overwrite_log: true, log_step: 1, split_output: false, parent_path: /tmp/tmp.T7RXg6yei8} io: loader: batch_size: 16 shuffle: false num_workers: 8 collate_fn: all dataset: name: larcv file_keys: [/pscratch/sd/d/dunepro/cuddandr/output/FSD_CosmicRun3/run-mlreco/FSD_CosmicRun3.flow2supera/LARCV/0000000/FSD_CosmicRun3.flow2supera.0000177.LARCV.root] schema: data: {parser: sparse3d, sparse_event: sparse3d_pcluster} seg_label: {parser: sparse3d, sparse_event: sparse3d_pcluster_semantics} ppn_label: {parser: particle_points, sparse_event: sparse3d_pcluster, particle_event: particle_pcluster, include_point_tagging: false} clust_label: {parser: cluster3d, cluster_event: cluster3d_pcluster, particle_event: particle_pcluster, neutrino_event: neutrino_mc_truth, sparse_semantics_event: sparse3d_pcluster_semantics, add_particle_info: true, clean_data: true} coord_label: {parser: particle_coords, particle_event: particle_pcluster, cluster_event: cluster3d_pcluster} particles: {parser: particle, particle_event: particle_pcluster, neutrino_event: neutrino_mc_truth, cluster_event: cluster3d_pcluster, skip_empty: true} neutrinos: {parser: neutrino, neutrino_event: neutrino_mc_truth, cluster_event: cluster3d_pcluster} meta: {parser: meta, sparse_event: sparse3d_pcluster} run_info: {parser: run_info, sparse_event: sparse3d_pcluster} trigger: {parser: trigger, trigger_event: trigger_base} flashes: {parser: flash, flash_event: opflash_light} writer: name: hdf5 file_name: /pscratch/sd/d/dunepro/cuddandr/output/FSD_CosmicRun3/tmp/run-mlreco/FSD_CosmicRun3.spine/FSD_CosmicRun3.spine.0000177.MLRECO_SPINE.hdf5 overwrite: true keys: [run_info, meta, trigger, flashes, points, points_label, depositions, depositions_label, reco_particles, truth_particles, reco_interactions, truth_interactions] model: name: full_chain weight_path: /dvs_ro/cfs/projectdirs/dune/users/drielsma/weights/ndlar_250505_snapshot.ckpt to_numpy: true network_input: {data: data, seg_label: seg_label} modules: chain: {deghosting: null, charge_rescaling: null, segmentation: uresnet, point_proposal: ppn, fragmentation: graph_spice, shower_aggregation: grappa, shower_primary: grappa, track_aggregation: grappa, particle_aggregation: null, inter_aggregation: grappa, particle_identification: grappa, primary_identification: grappa, orientation_identification: grappa, calibration: null} uresnet_ppn: uresnet: num_input: 1 num_classes: 5 filters: 32 depth: 5 reps: 2 allow_bias: false activation: {name: lrelu, negative_slope: 0.33} norm_layer: {name: batch_norm, eps: 0.0001, momentum: 0.01} ppn: {classify_endpoints: false} uresnet_ppn_loss: uresnet_loss: {balance_loss: false} ppn_loss: {mask_loss: CE, resolution: 5.0} graph_spice: shapes: [shower, track, michel, delta] use_raw_features: true invert: true make_clusters: true embedder: spatial_embedding_dim: 3 feature_embedding_dim: 16 occupancy_mode: softplus covariance_mode: softplus uresnet: num_input: 4 filters: 32 input_kernel: 5 depth: 5 reps: 2 spatial_size: 6144 allow_bias: false activation: {name: lrelu, negative_slope: 0.33} norm_layer: {name: batch_norm, eps: 0.0001, momentum: 0.01} kernel: {name: bilinear, num_features: 32} constructor: edge_threshold: 0.1 min_size: 3 label_edges: false graph: {name: radius, r: 1.9} orphan: {mode: radius, radius: 1.9, iterate: true, assign_all: true} graph_spice_loss: {name: edge, loss: binary_log_dice_ce} grappa_shower: nodes: source: cluster shapes: [shower, michel, delta] make_groups: true grouping_method: score graph: name: complete max_length: [500, 0, 500, 500, 0, 0, 0, 25, 0, 25] dist_algorithm: recursive node_encoder: {name: geo, use_numpy: true, add_value: true, add_shape: true, add_points: true, add_local_dirs: true, dir_max_dist: 5, add_local_dedxs: true, dedx_max_dist: 5} edge_encoder: {name: geo, use_numpy: true} gnn_model: name: meta node_feats: 33 edge_feats: 19 node_pred: 2 edge_pred: 2 edge_layer: name: mlp mlp: depth: 3 width: 64 activation: {name: lrelu, negative_slope: 0.1} normalization: batch_norm node_layer: name: mlp reduction: max attention: false message_mlp: depth: 3 width: 64 activation: {name: lrelu, negative_slope: 0.1} normalization: batch_norm aggr_mlp: depth: 3 width: 64 activation: {name: lrelu, negative_slope: 0.1} normalization: batch_norm grappa_shower_loss: node_loss: {name: shower_primary, use_closest: true, high_purity: true, use_group_pred: true} edge_loss: {name: channel, target: group, high_purity: true} grappa_track: nodes: source: cluster shapes: [track] make_groups: true grouping_method: score graph: {name: complete, max_length: 100, dist_algorithm: recursive} node_encoder: {name: geo, use_numpy: true, add_value: true, add_shape: false, add_points: true, add_local_dirs: true, dir_max_dist: 5, add_local_dedxs: true, dedx_max_dist: 5} edge_encoder: {name: geo, use_numpy: true} gnn_model: name: meta node_feats: 32 edge_feats: 19 edge_pred: 2 edge_layer: name: mlp mlp: depth: 3 width: 64 activation: {name: lrelu, negative_slope: 0.1} normalization: batch_norm node_layer: name: mlp reduction: max attention: false message_mlp: depth: 3 width: 64 activation: {name: lrelu, negative_slope: 0.1} normalization: batch_norm aggr_mlp: depth: 3 width: 64 activation: {name: lrelu, negative_slope: 0.1} normalization: batch_norm grappa_track_loss: edge_loss: {name: channel, target: group} grappa_inter: nodes: source: group shapes: [shower, track, michel, delta] min_size: -1 make_groups: true graph: name: complete max_length: [500, 500, 0, 0, 25, 25, 25, 0, 0, 0] dist_algorithm: recursive node_encoder: {name: geo, use_numpy: true, add_value: true, add_shape: true, add_points: true, add_local_dirs: true, dir_max_dist: 5, add_local_dedxs: true, dedx_max_dist: 5} edge_encoder: {name: geo, use_numpy: true} gnn_model: name: meta node_feats: 33 edge_feats: 19 node_pred: {type: 6, primary: 2, orient: 2} edge_pred: 2 edge_layer: name: mlp mlp: depth: 3 width: 128 activation: {name: lrelu, negative_slope: 0.1} normalization: batch_norm node_layer: name: mlp reduction: max attention: false message_mlp: depth: 3 width: 128 activation: {name: lrelu, negative_slope: 0.1} normalization: batch_norm aggr_mlp: depth: 3 width: 128 activation: {name: lrelu, negative_slope: 0.1} normalization: batch_norm grappa_inter_loss: node_loss: type: {name: class, target: pid, loss: ce, balance_loss: true} primary: {name: class, target: inter_primary, loss: ce, balance_loss: true} orient: {name: orient, loss: ce} edge_loss: {name: channel, target: interaction} build: {mode: both, units: cm, fragments: false, particles: true, interactions: true} post: shape_logic: {enforce_pid: true, enforce_primary: true, priority: 3} direction: {run_mode: reco, obj_type: particle, optimize: true, priority: 1, radius: 60} calo_ke: {run_mode: both, scaling: 1.0, shower_fudge: 1/0.83, priority: 1} csda_ke: {run_mode: reco, tracking_mode: step_next, segment_length: 5.0, priority: 1} mcs_ke: {run_mode: reco, tracking_mode: bin_pca, segment_length: 5.0} containment: {run_mode: both, detector: ndlar_single, margin: 5.0, mode: detector, priority: 1} vertex: {run_mode: reco, use_primaries: true, update_primaries: false, priority: 1} topology_threshold: run_mode: both ke_thresholds: {4: 50, default: 25} fiducial: {run_mode: both, detector: ndlar_single, margin: 15.0, mode: detector} children_count: {mode: shape} match: {match_mode: both, ghost: false, truth_point_mode: points, fragment: false, particle: true, interaction: true} Will load 1 file(s): - /pscratch/sd/d/dunepro/cuddandr/output/FSD_CosmicRun3/run-mlreco/FSD_CosmicRun3.flow2supera/LARCV/0000000/FSD_CosmicRun3.flow2supera.0000177.LARCV.root Loading tree sparse3d_pcluster Loading tree sparse3d_pcluster_semantics Loading tree particle_pcluster Loading tree cluster3d_pcluster Loading tree neutrino_mc_truth Loading tree trigger_base Loading tree opflash_light Total number of entries in the file(s): 2610 Total number of entries selected: 2610 Full chain configuration: deghosting : null charge_rescaling : null segmentation : uresnet point_proposal : ppn fragmentation : graph_spice shower_aggregation : grappa shower_primary : grappa track_aggregation : grappa particle_aggregation : null inter_aggregation : grappa particle_identification : grappa primary_identification : grappa orientation_identification : grappa calibration : null Restoring weights for module full_chain from /dvs_ro/cfs/projectdirs/dune/users/drielsma/weights/ndlar_250505_snapshot.ckpt... Done. Iter. 0 (epoch 0.006) @ 2025-08-21 10:20:02 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 8.53 s (80.96 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 1 (epoch 0.012) @ 2025-08-21 10:20:10 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.58 s (27.70 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 2 (epoch 0.018) @ 2025-08-21 10:20:11 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.53 s (42.84 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 3 (epoch 0.024) @ 2025-08-21 10:20:11 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.37 s (38.62 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 4 (epoch 0.030) @ 2025-08-21 10:20:12 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.49 s (36.80 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 5 (epoch 0.037) @ 2025-08-21 10:20:12 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.46 s (37.20 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 6 (epoch 0.043) @ 2025-08-21 10:20:13 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.46 s (44.39 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 7 (epoch 0.049) @ 2025-08-21 10:20:13 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.39 s (39.06 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 8 (epoch 0.055) @ 2025-08-21 10:20:14 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.41 s (38.75 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 9 (epoch 0.061) @ 2025-08-21 10:20:14 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.42 s (37.58 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 10 (epoch 0.067) @ 2025-08-21 10:20:14 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.52 s (40.59 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 11 (epoch 0.073) @ 2025-08-21 10:20:15 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.46 s (51.45 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 12 (epoch 0.079) @ 2025-08-21 10:20:15 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.49 s (36.47 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 13 (epoch 0.085) @ 2025-08-21 10:20:16 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.40 s (41.98 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 14 (epoch 0.091) @ 2025-08-21 10:20:16 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.42 s (36.53 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 15 (epoch 0.098) @ 2025-08-21 10:20:17 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.37 s (40.28 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 16 (epoch 0.104) @ 2025-08-21 10:20:17 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.37 s (39.27 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 17 (epoch 0.110) @ 2025-08-21 10:20:18 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.36 s (46.57 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 18 (epoch 0.116) @ 2025-08-21 10:20:18 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.47 s (31.04 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 19 (epoch 0.122) @ 2025-08-21 10:20:18 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.34 s (45.42 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 20 (epoch 0.128) @ 2025-08-21 10:20:19 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.44 s (35.76 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 21 (epoch 0.134) @ 2025-08-21 10:20:19 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.51 s (31.58 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 22 (epoch 0.140) @ 2025-08-21 10:20:20 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.37 s (41.59 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 23 (epoch 0.146) @ 2025-08-21 10:20:20 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.41 s (42.95 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 24 (epoch 0.152) @ 2025-08-21 10:20:20 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.42 s (41.81 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 25 (epoch 0.159) @ 2025-08-21 10:20:21 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.35 s (38.27 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 26 (epoch 0.165) @ 2025-08-21 10:20:21 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.46 s (32.83 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 27 (epoch 0.171) @ 2025-08-21 10:20:22 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.37 s (37.73 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 28 (epoch 0.177) @ 2025-08-21 10:20:22 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.41 s (34.90 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 29 (epoch 0.183) @ 2025-08-21 10:20:22 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.41 s (38.35 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 30 (epoch 0.189) @ 2025-08-21 10:20:23 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.38 s (37.74 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 31 (epoch 0.195) @ 2025-08-21 10:20:23 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.53 s (55.91 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 32 (epoch 0.201) @ 2025-08-21 10:20:24 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.38 s (37.90 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 33 (epoch 0.207) @ 2025-08-21 10:20:24 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.40 s (36.81 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 34 (epoch 0.213) @ 2025-08-21 10:20:25 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.43 s (36.31 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 35 (epoch 0.220) @ 2025-08-21 10:20:25 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.35 s (37.35 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 36 (epoch 0.226) @ 2025-08-21 10:20:25 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.41 s (36.27 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 37 (epoch 0.232) @ 2025-08-21 10:20:26 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.33 s (38.83 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 38 (epoch 0.238) @ 2025-08-21 10:20:26 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.29 s (41.18 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 39 (epoch 0.244) @ 2025-08-21 10:20:26 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.42 s (41.88 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 40 (epoch 0.250) @ 2025-08-21 10:20:27 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.49 s (41.24 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 41 (epoch 0.256) @ 2025-08-21 10:20:27 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.58 s (58.41 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 42 (epoch 0.262) @ 2025-08-21 10:20:28 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.49 s (41.80 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 43 (epoch 0.268) @ 2025-08-21 10:20:28 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.42 s (33.42 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 44 (epoch 0.274) @ 2025-08-21 10:20:29 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.48 s (41.05 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 45 (epoch 0.280) @ 2025-08-21 10:20:29 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.38 s (37.30 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 46 (epoch 0.287) @ 2025-08-21 10:20:30 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.37 s (38.62 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 47 (epoch 0.293) @ 2025-08-21 10:20:30 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.36 s (40.49 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 48 (epoch 0.299) @ 2025-08-21 10:20:31 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.54 s (53.78 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 49 (epoch 0.305) @ 2025-08-21 10:20:31 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.48 s (45.37 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 50 (epoch 0.311) @ 2025-08-21 10:20:32 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.49 s (43.71 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 51 (epoch 0.317) @ 2025-08-21 10:20:32 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.37 s (36.62 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 52 (epoch 0.323) @ 2025-08-21 10:20:32 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.34 s (36.62 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 53 (epoch 0.329) @ 2025-08-21 10:20:33 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.46 s (48.00 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 54 (epoch 0.335) @ 2025-08-21 10:20:33 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.39 s (44.61 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 55 (epoch 0.341) @ 2025-08-21 10:20:34 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.47 s (47.17 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 56 (epoch 0.348) @ 2025-08-21 10:20:34 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.46 s (42.71 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 57 (epoch 0.354) @ 2025-08-21 10:20:35 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.49 s (34.31 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 58 (epoch 0.360) @ 2025-08-21 10:20:35 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.42 s (33.49 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 59 (epoch 0.366) @ 2025-08-21 10:20:35 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.46 s (36.81 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 60 (epoch 0.372) @ 2025-08-21 10:20:36 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.43 s (35.06 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 61 (epoch 0.378) @ 2025-08-21 10:20:36 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.41 s (37.59 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 62 (epoch 0.384) @ 2025-08-21 10:20:37 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.41 s (39.41 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 63 (epoch 0.390) @ 2025-08-21 10:20:37 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.34 s (39.89 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 64 (epoch 0.396) @ 2025-08-21 10:20:38 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.49 s (28.13 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 65 (epoch 0.402) @ 2025-08-21 10:20:38 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.46 s (49.58 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 66 (epoch 0.409) @ 2025-08-21 10:20:39 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.55 s (53.66 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 67 (epoch 0.415) @ 2025-08-21 10:20:39 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.61 s (26.16 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 68 (epoch 0.421) @ 2025-08-21 10:20:40 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.46 s (35.23 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 69 (epoch 0.427) @ 2025-08-21 10:20:40 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 1.01 s (42.55 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 70 (epoch 0.433) @ 2025-08-21 10:20:41 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.47 s (52.45 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 71 (epoch 0.439) @ 2025-08-21 10:20:42 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.37 s (38.97 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 72 (epoch 0.445) @ 2025-08-21 10:20:42 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.43 s (36.80 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 73 (epoch 0.451) @ 2025-08-21 10:20:42 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.82 s (61.17 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 74 (epoch 0.457) @ 2025-08-21 10:20:43 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.76 s (65.43 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 75 (epoch 0.463) @ 2025-08-21 10:20:44 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.65 s (50.33 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 76 (epoch 0.470) @ 2025-08-21 10:20:45 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.44 s (37.98 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 77 (epoch 0.476) @ 2025-08-21 10:20:45 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.53 s (51.39 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 78 (epoch 0.482) @ 2025-08-21 10:20:46 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.48 s (44.28 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 79 (epoch 0.488) @ 2025-08-21 10:20:46 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.38 s (37.95 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 80 (epoch 0.494) @ 2025-08-21 10:20:47 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.39 s (37.11 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 81 (epoch 0.500) @ 2025-08-21 10:20:47 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.51 s (35.80 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 82 (epoch 0.506) @ 2025-08-21 10:20:47 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.72 s (56.22 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 83 (epoch 0.512) @ 2025-08-21 10:20:48 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.39 s (46.28 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 84 (epoch 0.518) @ 2025-08-21 10:20:49 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.52 s (26.90 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 85 (epoch 0.524) @ 2025-08-21 10:20:49 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.39 s (38.17 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 86 (epoch 0.530) @ 2025-08-21 10:20:50 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.47 s (31.94 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 87 (epoch 0.537) @ 2025-08-21 10:20:50 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.44 s (44.49 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 88 (epoch 0.543) @ 2025-08-21 10:20:50 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.39 s (38.13 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 89 (epoch 0.549) @ 2025-08-21 10:20:51 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.42 s (37.15 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 90 (epoch 0.555) @ 2025-08-21 10:20:51 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.55 s (55.35 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 91 (epoch 0.561) @ 2025-08-21 10:20:52 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.42 s (40.02 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 92 (epoch 0.567) @ 2025-08-21 10:20:52 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.49 s (31.99 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 93 (epoch 0.573) @ 2025-08-21 10:20:53 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.44 s (40.58 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 94 (epoch 0.579) @ 2025-08-21 10:20:53 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.39 s (37.74 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 95 (epoch 0.585) @ 2025-08-21 10:20:54 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.45 s (33.26 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 96 (epoch 0.591) @ 2025-08-21 10:20:54 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.42 s (46.56 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 97 (epoch 0.598) @ 2025-08-21 10:20:54 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.43 s (39.84 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 98 (epoch 0.604) @ 2025-08-21 10:20:55 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.51 s (32.23 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 99 (epoch 0.610) @ 2025-08-21 10:20:55 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.52 s (42.38 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 100 (epoch 0.616) @ 2025-08-21 10:20:56 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.77 s (54.31 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 101 (epoch 0.622) @ 2025-08-21 10:20:57 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.39 s (40.20 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 102 (epoch 0.628) @ 2025-08-21 10:20:57 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.40 s (40.55 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 103 (epoch 0.634) @ 2025-08-21 10:20:58 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.46 s (46.01 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 104 (epoch 0.640) @ 2025-08-21 10:20:58 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.41 s (42.14 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 105 (epoch 0.646) @ 2025-08-21 10:20:58 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.41 s (36.30 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 106 (epoch 0.652) @ 2025-08-21 10:20:59 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.36 s (37.96 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 107 (epoch 0.659) @ 2025-08-21 10:20:59 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.42 s (42.37 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 108 (epoch 0.665) @ 2025-08-21 10:21:00 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.65 s (58.97 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 109 (epoch 0.671) @ 2025-08-21 10:21:00 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.35 s (39.91 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 110 (epoch 0.677) @ 2025-08-21 10:21:01 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.67 s (57.67 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 111 (epoch 0.683) @ 2025-08-21 10:21:01 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.39 s (38.54 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 112 (epoch 0.689) @ 2025-08-21 10:21:02 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.54 s (51.34 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 113 (epoch 0.695) @ 2025-08-21 10:21:02 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.47 s (49.66 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 114 (epoch 0.701) @ 2025-08-21 10:21:03 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.50 s (52.49 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 115 (epoch 0.707) @ 2025-08-21 10:21:03 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.78 s (64.22 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 116 (epoch 0.713) @ 2025-08-21 10:21:04 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.62 s (53.55 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 117 (epoch 0.720) @ 2025-08-21 10:21:05 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.40 s (38.89 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 118 (epoch 0.726) @ 2025-08-21 10:21:05 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.43 s (34.10 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 119 (epoch 0.732) @ 2025-08-21 10:21:05 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.34 s (40.41 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 120 (epoch 0.738) @ 2025-08-21 10:21:06 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.84 s (46.64 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 121 (epoch 0.744) @ 2025-08-21 10:21:07 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.58 s (59.22 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 122 (epoch 0.750) @ 2025-08-21 10:21:07 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.49 s (43.88 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 123 (epoch 0.756) @ 2025-08-21 10:21:08 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.53 s (29.31 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 124 (epoch 0.762) @ 2025-08-21 10:21:08 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.53 s (50.69 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 125 (epoch 0.768) @ 2025-08-21 10:21:09 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.36 s (41.26 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 126 (epoch 0.774) @ 2025-08-21 10:21:09 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.37 s (38.10 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 127 (epoch 0.780) @ 2025-08-21 10:21:10 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.32 s (40.50 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 128 (epoch 0.787) @ 2025-08-21 10:21:10 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.44 s (44.06 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 129 (epoch 0.793) @ 2025-08-21 10:21:10 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.43 s (43.23 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 130 (epoch 0.799) @ 2025-08-21 10:21:11 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.40 s (34.16 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 131 (epoch 0.805) @ 2025-08-21 10:21:11 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.75 s (65.63 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 132 (epoch 0.811) @ 2025-08-21 10:21:12 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.49 s (33.29 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 133 (epoch 0.817) @ 2025-08-21 10:21:12 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.44 s (34.21 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 134 (epoch 0.823) @ 2025-08-21 10:21:13 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.34 s (38.85 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 135 (epoch 0.829) @ 2025-08-21 10:21:13 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.39 s (38.94 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 136 (epoch 0.835) @ 2025-08-21 10:21:14 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.49 s (32.33 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 137 (epoch 0.841) @ 2025-08-21 10:21:14 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.44 s (35.31 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 138 (epoch 0.848) @ 2025-08-21 10:21:15 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.47 s (38.34 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 139 (epoch 0.854) @ 2025-08-21 10:21:15 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.75 s (61.41 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 140 (epoch 0.860) @ 2025-08-21 10:21:16 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.47 s (41.40 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 141 (epoch 0.866) @ 2025-08-21 10:21:16 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.35 s (41.03 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 142 (epoch 0.872) @ 2025-08-21 10:21:17 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.44 s (37.26 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 143 (epoch 0.878) @ 2025-08-21 10:21:17 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.36 s (41.42 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 144 (epoch 0.884) @ 2025-08-21 10:21:17 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.43 s (36.12 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 145 (epoch 0.890) @ 2025-08-21 10:21:18 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.60 s (57.05 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 146 (epoch 0.896) @ 2025-08-21 10:21:18 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.42 s (35.83 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 147 (epoch 0.902) @ 2025-08-21 10:21:19 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.55 s (35.41 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 148 (epoch 0.909) @ 2025-08-21 10:21:19 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.39 s (38.11 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 149 (epoch 0.915) @ 2025-08-21 10:21:20 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.42 s (35.76 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 150 (epoch 0.921) @ 2025-08-21 10:21:20 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.40 s (35.47 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 151 (epoch 0.927) @ 2025-08-21 10:21:21 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.35 s (38.56 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 152 (epoch 0.933) @ 2025-08-21 10:21:21 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.42 s (37.14 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 153 (epoch 0.939) @ 2025-08-21 10:21:21 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.42 s (36.81 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 154 (epoch 0.945) @ 2025-08-21 10:21:22 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.34 s (39.78 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 155 (epoch 0.951) @ 2025-08-21 10:21:22 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.44 s (35.58 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 156 (epoch 0.957) @ 2025-08-21 10:21:23 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.39 s (36.06 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 157 (epoch 0.963) @ 2025-08-21 10:21:23 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.38 s (37.71 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 158 (epoch 0.970) @ 2025-08-21 10:21:23 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.44 s (35.61 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 159 (epoch 0.976) @ 2025-08-21 10:21:24 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.39 s (39.40 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 160 (epoch 0.982) @ 2025-08-21 10:21:24 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.41 s (39.54 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 161 (epoch 0.988) @ 2025-08-21 10:21:25 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.51 s (36.01 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | Iter. 162 (epoch 0.994) @ 2025-08-21 10:21:25 | Time (inference) | GPU memory | Loss | Accuracy | |---------------------+---------------------+----------+----------| | 0.42 s (36.73 %) | 0.23 GB (0.53 %) | -1.000 | -1.000 | *** Break *** segmentation violation =========================================================== There was a crash. This is the entire stack trace of all threads: =========================================================== Thread 59 (Thread 0x7f37664bb640 (LWP 954713) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 58 (Thread 0x7f3766cbc640 (LWP 954712) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 57 (Thread 0x7f37674bd640 (LWP 954711) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 56 (Thread 0x7f3767cbe640 (LWP 954710) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 55 (Thread 0x7f37684bf640 (LWP 954709) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 54 (Thread 0x7f3768cc0640 (LWP 954708) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 53 (Thread 0x7f37694c1640 (LWP 954707) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 52 (Thread 0x7f3769cc2640 (LWP 954706) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 51 (Thread 0x7f376a4c3640 (LWP 954705) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 50 (Thread 0x7f376acc4640 (LWP 954704) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 49 (Thread 0x7f376b4c5640 (LWP 954703) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 48 (Thread 0x7f376bcc6640 (LWP 954702) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 47 (Thread 0x7f376c4c7640 (LWP 954701) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 46 (Thread 0x7f376ccc8640 (LWP 954700) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 45 (Thread 0x7f376d4c9640 (LWP 954699) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 44 (Thread 0x7f376dcca640 (LWP 954698) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 43 (Thread 0x7f376e4cb640 (LWP 954697) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 42 (Thread 0x7f376eccc640 (LWP 954696) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 41 (Thread 0x7f376f4cd640 (LWP 954695) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 40 (Thread 0x7f376fcce640 (LWP 954694) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 39 (Thread 0x7f37704cf640 (LWP 954693) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 38 (Thread 0x7f3770cd0640 (LWP 954692) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 37 (Thread 0x7f37714d1640 (LWP 954691) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 36 (Thread 0x7f3771cd2640 (LWP 954690) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 35 (Thread 0x7f37724d3640 (LWP 954689) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 34 (Thread 0x7f3772cd4640 (LWP 954688) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 33 (Thread 0x7f37734d5640 (LWP 954687) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 32 (Thread 0x7f3773cd6640 (LWP 954686) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 31 (Thread 0x7f37744d7640 (LWP 954685) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 30 (Thread 0x7f3774cd8640 (LWP 954684) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 29 (Thread 0x7f37754d9640 (LWP 954683) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 28 (Thread 0x7f3797fff640 (LWP 954671) "python3"): #0 __futex_abstimed_wait_common64 (private=, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3790000e00) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3790000e00) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3790000e00, expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=) at ./nptl/futex-internal.c:139 #3 0x00007f3913e71bdf in do_futex_wait (sem=sem entry=0x7f3790000e00, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111 #4 0x00007f3913e71c78 in __new_sem_wait_slow64 (sem=0x7f3790000e00, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183 #5 0x00005600a193add0 in PyThread_acquire_lock_timed () #6 0x00005600a198ea64 in ?? () #7 0x00005600a1979de7 in ?? () #8 0x00005600a196397f in _PyEval_EvalFrameDefault () #9 0x00005600a19864d1 in ?? () #10 0x00005600a1963790 in _PyEval_EvalFrameDefault () #11 0x00005600a197938c in _PyFunction_Vectorcall () #12 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #13 0x00005600a197938c in _PyFunction_Vectorcall () #14 0x00005600a196397f in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00005600a196397f in _PyEval_EvalFrameDefault () #17 0x00005600a1986731 in ?? () #18 0x00005600a1a9b7ca in ?? () #19 0x00005600a1a91138 in ?? () #20 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #21 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 27 (Thread 0x7f37b4ffd640 (LWP 954669) "python3"): #0 __futex_abstimed_wait_common64 (private=, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f379c001020) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=, abstime=0x0, clockid=0, expected=0, futex_word=0x7f379c001020) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f379c001020, expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=) at ./nptl/futex-internal.c:139 #3 0x00007f3913e71bdf in do_futex_wait (sem=sem entry=0x7f379c001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111 #4 0x00007f3913e71c78 in __new_sem_wait_slow64 (sem=0x7f379c001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183 #5 0x00005600a193add0 in PyThread_acquire_lock_timed () #6 0x00005600a198ea64 in ?? () #7 0x00005600a1979de7 in ?? () #8 0x00005600a196397f in _PyEval_EvalFrameDefault () #9 0x00005600a19864d1 in ?? () #10 0x00005600a1963790 in _PyEval_EvalFrameDefault () #11 0x00005600a197938c in _PyFunction_Vectorcall () #12 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #13 0x00005600a197938c in _PyFunction_Vectorcall () #14 0x00005600a196397f in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00005600a196397f in _PyEval_EvalFrameDefault () #17 0x00005600a1986731 in ?? () #18 0x00005600a1a9b7ca in ?? () #19 0x00005600a1a91138 in ?? () #20 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #21 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 26 (Thread 0x7f37b57fe640 (LWP 954667) "python3"): #0 __futex_abstimed_wait_common64 (private=, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3798001020) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3798001020) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3798001020, expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=) at ./nptl/futex-internal.c:139 #3 0x00007f3913e71bdf in do_futex_wait (sem=sem entry=0x7f3798001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111 #4 0x00007f3913e71c78 in __new_sem_wait_slow64 (sem=0x7f3798001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183 #5 0x00005600a193add0 in PyThread_acquire_lock_timed () #6 0x00005600a198ea64 in ?? () #7 0x00005600a1979de7 in ?? () #8 0x00005600a196397f in _PyEval_EvalFrameDefault () #9 0x00005600a19864d1 in ?? () #10 0x00005600a1963790 in _PyEval_EvalFrameDefault () #11 0x00005600a197938c in _PyFunction_Vectorcall () #12 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #13 0x00005600a197938c in _PyFunction_Vectorcall () #14 0x00005600a196397f in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00005600a196397f in _PyEval_EvalFrameDefault () #17 0x00005600a1986731 in ?? () #18 0x00005600a1a9b7ca in ?? () #19 0x00005600a1a91138 in ?? () #20 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #21 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 25 (Thread 0x7f37b5fff640 (LWP 954666) "python3"): #0 __futex_abstimed_wait_common64 (private=, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f37a4001020) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=, abstime=0x0, clockid=0, expected=0, futex_word=0x7f37a4001020) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f37a4001020, expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=) at ./nptl/futex-internal.c:139 #3 0x00007f3913e71bdf in do_futex_wait (sem=sem entry=0x7f37a4001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111 #4 0x00007f3913e71c78 in __new_sem_wait_slow64 (sem=0x7f37a4001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183 #5 0x00005600a193add0 in PyThread_acquire_lock_timed () #6 0x00005600a198ea64 in ?? () #7 0x00005600a1979de7 in ?? () #8 0x00005600a196397f in _PyEval_EvalFrameDefault () #9 0x00005600a19864d1 in ?? () #10 0x00005600a1963790 in _PyEval_EvalFrameDefault () #11 0x00005600a197938c in _PyFunction_Vectorcall () #12 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #13 0x00005600a197938c in _PyFunction_Vectorcall () #14 0x00005600a196397f in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00005600a196397f in _PyEval_EvalFrameDefault () #17 0x00005600a1986731 in ?? () #18 0x00005600a1a9b7ca in ?? () #19 0x00005600a1a91138 in ?? () #20 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #21 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 24 (Thread 0x7f37e4a10640 (LWP 954665) "python3"): #0 __futex_abstimed_wait_common64 (private=, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f37a0001020) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=, abstime=0x0, clockid=0, expected=0, futex_word=0x7f37a0001020) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f37a0001020, expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=) at ./nptl/futex-internal.c:139 #3 0x00007f3913e71bdf in do_futex_wait (sem=sem entry=0x7f37a0001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111 #4 0x00007f3913e71c78 in __new_sem_wait_slow64 (sem=0x7f37a0001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183 #5 0x00005600a193add0 in PyThread_acquire_lock_timed () #6 0x00005600a198ea64 in ?? () #7 0x00005600a1979de7 in ?? () #8 0x00005600a196397f in _PyEval_EvalFrameDefault () #9 0x00005600a19864d1 in ?? () #10 0x00005600a1963790 in _PyEval_EvalFrameDefault () #11 0x00005600a197938c in _PyFunction_Vectorcall () #12 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #13 0x00005600a197938c in _PyFunction_Vectorcall () #14 0x00005600a196397f in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00005600a196397f in _PyEval_EvalFrameDefault () #17 0x00005600a1986731 in ?? () #18 0x00005600a1a9b7ca in ?? () #19 0x00005600a1a91138 in ?? () #20 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #21 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 23 (Thread 0x7f37e5211640 (LWP 954664) "python3"): #0 __futex_abstimed_wait_common64 (private=, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f37ac001020) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=, abstime=0x0, clockid=0, expected=0, futex_word=0x7f37ac001020) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f37ac001020, expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=) at ./nptl/futex-internal.c:139 #3 0x00007f3913e71bdf in do_futex_wait (sem=sem entry=0x7f37ac001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111 #4 0x00007f3913e71c78 in __new_sem_wait_slow64 (sem=0x7f37ac001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183 #5 0x00005600a193add0 in PyThread_acquire_lock_timed () #6 0x00005600a198ea64 in ?? () #7 0x00005600a1979de7 in ?? () #8 0x00005600a196397f in _PyEval_EvalFrameDefault () #9 0x00005600a19864d1 in ?? () #10 0x00005600a1963790 in _PyEval_EvalFrameDefault () #11 0x00005600a197938c in _PyFunction_Vectorcall () #12 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #13 0x00005600a197938c in _PyFunction_Vectorcall () #14 0x00005600a196397f in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00005600a196397f in _PyEval_EvalFrameDefault () #17 0x00005600a1986731 in ?? () #18 0x00005600a1a9b7ca in ?? () #19 0x00005600a1a91138 in ?? () #20 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #21 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 22 (Thread 0x7f37e5a12640 (LWP 954663) "python3"): #0 __futex_abstimed_wait_common64 (private=, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f37a8001020) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=, abstime=0x0, clockid=0, expected=0, futex_word=0x7f37a8001020) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f37a8001020, expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=) at ./nptl/futex-internal.c:139 #3 0x00007f3913e71bdf in do_futex_wait (sem=sem entry=0x7f37a8001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111 #4 0x00007f3913e71c78 in __new_sem_wait_slow64 (sem=0x7f37a8001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183 #5 0x00005600a193add0 in PyThread_acquire_lock_timed () #6 0x00005600a198ea64 in ?? () #7 0x00005600a1979de7 in ?? () #8 0x00005600a196397f in _PyEval_EvalFrameDefault () #9 0x00005600a19864d1 in ?? () #10 0x00005600a1963790 in _PyEval_EvalFrameDefault () #11 0x00005600a197938c in _PyFunction_Vectorcall () #12 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #13 0x00005600a197938c in _PyFunction_Vectorcall () #14 0x00005600a196397f in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00005600a196397f in _PyEval_EvalFrameDefault () #17 0x00005600a1986731 in ?? () #18 0x00005600a1a9b7ca in ?? () #19 0x00005600a1a91138 in ?? () #20 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #21 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 21 (Thread 0x7f37e6213640 (LWP 954636) "python3"): #0 __futex_abstimed_wait_common64 (private=, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f37b0001020) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=, abstime=0x0, clockid=0, expected=0, futex_word=0x7f37b0001020) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f37b0001020, expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=) at ./nptl/futex-internal.c:139 #3 0x00007f3913e71bdf in do_futex_wait (sem=sem entry=0x7f37b0001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111 #4 0x00007f3913e71c78 in __new_sem_wait_slow64 (sem=0x7f37b0001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183 #5 0x00005600a193add0 in PyThread_acquire_lock_timed () #6 0x00005600a198ea64 in ?? () #7 0x00005600a1979de7 in ?? () #8 0x00005600a196397f in _PyEval_EvalFrameDefault () #9 0x00005600a19864d1 in ?? () #10 0x00005600a1963790 in _PyEval_EvalFrameDefault () #11 0x00005600a197938c in _PyFunction_Vectorcall () #12 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #13 0x00005600a197938c in _PyFunction_Vectorcall () #14 0x00005600a196397f in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00005600a196397f in _PyEval_EvalFrameDefault () #17 0x00005600a1986731 in ?? () #18 0x00005600a1a9b7ca in ?? () #19 0x00005600a1a91138 in ?? () #20 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #21 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 20 (Thread 0x7f37e7e55640 (LWP 952858) "cuda-EvtHandlr"): #0 0x00007f3913eedbcf in __GI___poll (fds=0x7f37e0000c80, nfds=13, timeout=100) at ../sysdeps/unix/sysv/linux/poll.c:29 #1 0x00007f3858aaa44f in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1 #2 0x00007f3858b6e80f in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1 #3 0x00007f3858aa2143 in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1 #4 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #5 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 19 (Thread 0x7f3810947640 (LWP 951379) "cuda0000340000e"): #0 0x00007f3913eedbcf in __GI___poll (fds=0x5600b177d990, nfds=3, timeout=-1) at ../sysdeps/unix/sysv/linux/poll.c:29 #1 0x00007f3858aaa44f in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1 #2 0x00007f3858b6e80f in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1 #3 0x00007f3858aa2143 in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1 #4 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #5 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 18 (Thread 0x7f381ca57640 (LWP 951376) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 17 (Thread 0x7f381d258640 (LWP 951375) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 16 (Thread 0x7f381da59640 (LWP 951374) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 15 (Thread 0x7f381e25a640 (LWP 951373) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 14 (Thread 0x7f381ea5b640 (LWP 951372) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 13 (Thread 0x7f381f25c640 (LWP 951371) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 12 (Thread 0x7f381fa5d640 (LWP 951370) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 11 (Thread 0x7f382025e640 (LWP 951369) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 10 (Thread 0x7f3820a5f640 (LWP 951368) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 9 (Thread 0x7f3821260640 (LWP 951367) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 8 (Thread 0x7f3821a61640 (LWP 951366) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 7 (Thread 0x7f3822262640 (LWP 951365) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 6 (Thread 0x7f3822a63640 (LWP 951364) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 5 (Thread 0x7f3823264640 (LWP 951363) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 4 (Thread 0x7f3823a65640 (LWP 951362) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 3 (Thread 0x7f3824266640 (LWP 951361) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 2 (Thread 0x7f382fb62640 (LWP 951351) "python3"): #0 0x00007f3913ef063d in __GI___select (nfds=0, readfds=0x0, writefds=0x0, exceptfds=0x0, timeout=0x7f382fb615d0) at ../sysdeps/unix/sysv/linux/select.c:69 #1 0x00005600a1a832f6 in ?? () #2 0x00005600a1978534 in ?? () #3 0x00005600a1968702 in _PyEval_EvalFrameDefault () #4 0x00005600a197938c in _PyFunction_Vectorcall () #5 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #6 0x00005600a197938c in _PyFunction_Vectorcall () #7 0x00005600a196397f in _PyEval_EvalFrameDefault () #8 0x00005600a197938c in _PyFunction_Vectorcall () #9 0x00005600a196397f in _PyEval_EvalFrameDefault () #10 0x00005600a1986731 in ?? () #11 0x00005600a1a9b7ca in ?? () #12 0x00005600a1a91138 in ?? () #13 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #14 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 1 (Thread 0x7f3913dd0740 (LWP 951334) "python3"): #0 0x00007f3913ebf42f in __GI___wait4 (pid=961972, stat_loc=stat_loc entry=0x7ffd77fecd68, options=options entry=0, usage=usage entry=0x0) at ../sysdeps/unix/sysv/linux/wait4.c:30 #1 0x00007f3913ebf3ab in __GI___waitpid (pid=, stat_loc=stat_loc entry=0x7ffd77fecd68, options=options entry=0) at ./posix/waitpid.c:38 #2 0x00007f3913e25bdb in do_system (line=) at ../sysdeps/posix/system.c:171 #3 0x00007f3846e0fc84 in TUnixSystem::StackTrace() () from /app/root/lib/libCore.so #4 0x00007f38470b7ff3 in (anonymous namespace)::TExceptionHandlerImp::HandleException(int) () from /app/root/lib/libcppyy_backend3_10.so #5 0x00007f3846e0ce19 in TUnixSystem::DispatchSignals(ESignals) () from /app/root/lib/libCore.so #6 #7 0x00007f382cb5f9f6 in minkowski::CoordinateMapGPU::stride(std::vector > const&) const () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #8 0x00007f382cb9b559 in minkowski::CoordinateMapManager::stride(std::pair >, std::string> const&, std::vector > const&, std::string) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #9 0x00007f382cb2207a in at::Tensor minkowski::ConvolutionForwardGPU(at::Tensor const&, at::Tensor const&, std::vector > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #10 0x00007f382cc8b2d1 in pybind11::cpp_function::initialize > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*), at::Tensor, at::Tensor const&, at::Tensor const&, std::vector > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*, pybind11::name, pybind11::scope, pybind11::sibling, pybind11::call_guard >(at::Tensor (*&)(at::Tensor const&, at::Tensor const&, std::vector > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*), at::Tensor (*)(at::Tensor const&, at::Tensor const&, std::vector > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*), pybind11::name const&, pybind11::scope const&, pybind11::sibling const&, pybind11::call_guard const&)::{lambda(pybind11::detail::function_call&)#3}::_FUN(pybind11::detail::function_call&) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #11 0x00007f382cba6567 in pybind11::cpp_function::dispatcher(_object*, _object*, _object*) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #12 0x00005600a1978b32 in ?? () #13 0x00005600a196f39b in _PyObject_MakeTpCall () #14 0x00005600a1968a97 in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00007f3912244249 in THPFunction_apply(_object*, _object*) () from /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_python.so #17 0x00005600a1978b58 in ?? () #18 0x00005600a198710b in PyObject_Call () #19 0x00005600a19696ef in _PyEval_EvalFrameDefault () #20 0x00005600a19864d1 in ?? () #21 0x00005600a1968702 in _PyEval_EvalFrameDefault () #22 0x00005600a198661e in ?? () #23 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #24 0x00005600a198661e in ?? () #25 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #26 0x00005600a196e574 in _PyObject_FastCallDictTstate () #27 0x00005600a198362c in _PyObject_Call_Prepend () #28 0x00005600a1a8b464 in ?? () #29 0x00005600a196f39b in _PyObject_MakeTpCall () #30 0x00005600a1968a97 in _PyEval_EvalFrameDefault () #31 0x00005600a198661e in ?? () #32 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #33 0x00005600a198661e in ?? () #34 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #35 0x00005600a196e574 in _PyObject_FastCallDictTstate () #36 0x00005600a198362c in _PyObject_Call_Prepend () #37 0x00005600a1a8b464 in ?? () #38 0x00005600a196f39b in _PyObject_MakeTpCall () #39 0x00005600a1968a97 in _PyEval_EvalFrameDefault () #40 0x00005600a198661e in ?? () #41 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #42 0x00005600a198661e in ?? () #43 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #44 0x00005600a196e574 in _PyObject_FastCallDictTstate () #45 0x00005600a198362c in _PyObject_Call_Prepend () #46 0x00005600a1a8b464 in ?? () #47 0x00005600a196f39b in _PyObject_MakeTpCall () #48 0x00005600a1969009 in _PyEval_EvalFrameDefault () #49 0x00005600a198661e in ?? () #50 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #51 0x00005600a198661e in ?? () #52 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #53 0x00005600a196e574 in _PyObject_FastCallDictTstate () #54 0x00005600a198362c in _PyObject_Call_Prepend () #55 0x00005600a1a8b464 in ?? () #56 0x00005600a196f39b in _PyObject_MakeTpCall () #57 0x00005600a1969009 in _PyEval_EvalFrameDefault () #58 0x00005600a198661e in ?? () #59 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #60 0x00005600a198661e in ?? () #61 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #62 0x00005600a196e574 in _PyObject_FastCallDictTstate () #63 0x00005600a198362c in _PyObject_Call_Prepend () #64 0x00005600a1a8b464 in ?? () #65 0x00005600a196f39b in _PyObject_MakeTpCall () #66 0x00005600a1969009 in _PyEval_EvalFrameDefault () #67 0x00005600a198661e in ?? () #68 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #69 0x00005600a198661e in ?? () #70 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #71 0x00005600a196e574 in _PyObject_FastCallDictTstate () #72 0x00005600a198362c in _PyObject_Call_Prepend () #73 0x00005600a1a8b464 in ?? () #74 0x00005600a196f39b in _PyObject_MakeTpCall () #75 0x00005600a1969009 in _PyEval_EvalFrameDefault () #76 0x00005600a19864d1 in ?? () #77 0x00005600a1968702 in _PyEval_EvalFrameDefault () #78 0x00005600a19864d1 in ?? () #79 0x00005600a1987172 in PyObject_Call () #80 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #81 0x00005600a19864d1 in ?? () #82 0x00005600a1987172 in PyObject_Call () #83 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #84 0x00005600a197938c in _PyFunction_Vectorcall () #85 0x00005600a196e61d in _PyObject_FastCallDictTstate () #86 0x00005600a198362c in _PyObject_Call_Prepend () #87 0x00005600a1a8b464 in ?? () #88 0x00005600a198710b in PyObject_Call () #89 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #90 0x00005600a197938c in _PyFunction_Vectorcall () #91 0x00005600a196397f in _PyEval_EvalFrameDefault () #92 0x00005600a197938c in _PyFunction_Vectorcall () #93 0x00005600a196e61d in _PyObject_FastCallDictTstate () #94 0x00005600a198362c in _PyObject_Call_Prepend () #95 0x00005600a1a8b464 in ?? () #96 0x00005600a196f39b in _PyObject_MakeTpCall () #97 0x00005600a196999e in _PyEval_EvalFrameDefault () #98 0x00005600a19864d1 in ?? () #99 0x00005600a19649ab in _PyEval_EvalFrameDefault () #100 0x00005600a197938c in _PyFunction_Vectorcall () #101 0x00005600a196397f in _PyEval_EvalFrameDefault () #102 0x00005600a197938c in _PyFunction_Vectorcall () #103 0x00005600a1963790 in _PyEval_EvalFrameDefault () #104 0x00005600a197938c in _PyFunction_Vectorcall () #105 0x00005600a1963790 in _PyEval_EvalFrameDefault () #106 0x00005600a197938c in _PyFunction_Vectorcall () #107 0x00005600a1963790 in _PyEval_EvalFrameDefault () #108 0x00005600a197938c in _PyFunction_Vectorcall () #109 0x00005600a1963790 in _PyEval_EvalFrameDefault () #110 0x00005600a1a48a16 in ?? () #111 0x00005600a1a488e6 in PyEval_EvalCode () #112 0x00005600a1a6f108 in ?? () #113 0x00005600a1a6990f in ?? () #114 0x00005600a1a6eea5 in ?? () #115 0x00005600a1a6e488 in _PyRun_SimpleFileObject () #116 0x00005600a1a6e167 in _PyRun_AnyFileObject () #117 0x00005600a1a628ae in Py_RunMain () #118 0x00005600a1a3c93d in Py_BytesMain () #119 0x00007f3913dfed90 in __libc_start_call_main (main=main entry=0x5600a1a3c900, argc=argc entry=10, argv=argv entry=0x7ffd77ff6528) at ../sysdeps/nptl/libc_start_call_main.h:58 #120 0x00007f3913dfee40 in __libc_start_main_impl (main=0x5600a1a3c900, argc=10, argv=0x7ffd77ff6528, init=, fini=, rtld_fini=, stack_end=0x7ffd77ff6518) at ../csu/libc-start.c:392 #121 0x00005600a1a3c835 in _start () =========================================================== The lines below might hint at the cause of the crash. You may get help by asking at the ROOT forum https://root.cern/forum Only if you are really convinced it is a bug in ROOT then please submit a report at https://root.cern/bugs Please post the ENTIRE stack trace from above as an attachment in addition to anything else that might help us fixing this issue. =========================================================== #7 0x00007f382cb5f9f6 in minkowski::CoordinateMapGPU::stride(std::vector > const&) const () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #8 0x00007f382cb9b559 in minkowski::CoordinateMapManager::stride(std::pair >, std::string> const&, std::vector > const&, std::string) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #9 0x00007f382cb2207a in at::Tensor minkowski::ConvolutionForwardGPU(at::Tensor const&, at::Tensor const&, std::vector > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #10 0x00007f382cc8b2d1 in pybind11::cpp_function::initialize > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*), at::Tensor, at::Tensor const&, at::Tensor const&, std::vector > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*, pybind11::name, pybind11::scope, pybind11::sibling, pybind11::call_guard >(at::Tensor (*&)(at::Tensor const&, at::Tensor const&, std::vector > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*), at::Tensor (*)(at::Tensor const&, at::Tensor const&, std::vector > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*), pybind11::name const&, pybind11::scope const&, pybind11::sibling const&, pybind11::call_guard const&)::{lambda(pybind11::detail::function_call&)#3}::_FUN(pybind11::detail::function_call&) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #11 0x00007f382cba6567 in pybind11::cpp_function::dispatcher(_object*, _object*, _object*) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #12 0x00005600a1978b32 in ?? () #13 0x00005600a196f39b in _PyObject_MakeTpCall () #14 0x00005600a1968a97 in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00007f3912244249 in THPFunction_apply(_object*, _object*) () from /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_python.so #17 0x00005600a1978b58 in ?? () #18 0x00005600a198710b in PyObject_Call () #19 0x00005600a19696ef in _PyEval_EvalFrameDefault () #20 0x00005600a19864d1 in ?? () #21 0x00005600a1968702 in _PyEval_EvalFrameDefault () #22 0x00005600a198661e in ?? () #23 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #24 0x00005600a198661e in ?? () #25 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #26 0x00005600a196e574 in _PyObject_FastCallDictTstate () #27 0x00005600a198362c in _PyObject_Call_Prepend () #28 0x00005600a1a8b464 in ?? () #29 0x00005600a196f39b in _PyObject_MakeTpCall () #30 0x00005600a1968a97 in _PyEval_EvalFrameDefault () #31 0x00005600a198661e in ?? () #32 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #33 0x00005600a198661e in ?? () #34 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #35 0x00005600a196e574 in _PyObject_FastCallDictTstate () #36 0x00005600a198362c in _PyObject_Call_Prepend () #37 0x00005600a1a8b464 in ?? () #38 0x00005600a196f39b in _PyObject_MakeTpCall () #39 0x00005600a1968a97 in _PyEval_EvalFrameDefault () #40 0x00005600a198661e in ?? () #41 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #42 0x00005600a198661e in ?? () #43 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #44 0x00005600a196e574 in _PyObject_FastCallDictTstate () #45 0x00005600a198362c in _PyObject_Call_Prepend () #46 0x00005600a1a8b464 in ?? () #47 0x00005600a196f39b in _PyObject_MakeTpCall () #48 0x00005600a1969009 in _PyEval_EvalFrameDefault () #49 0x00005600a198661e in ?? () #50 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #51 0x00005600a198661e in ?? () #52 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #53 0x00005600a196e574 in _PyObject_FastCallDictTstate () #54 0x00005600a198362c in _PyObject_Call_Prepend () #55 0x00005600a1a8b464 in ?? () #56 0x00005600a196f39b in _PyObject_MakeTpCall () #57 0x00005600a1969009 in _PyEval_EvalFrameDefault () #58 0x00005600a198661e in ?? () #59 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #60 0x00005600a198661e in ?? () #61 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #62 0x00005600a196e574 in _PyObject_FastCallDictTstate () #63 0x00005600a198362c in _PyObject_Call_Prepend () #64 0x00005600a1a8b464 in ?? () #65 0x00005600a196f39b in _PyObject_MakeTpCall () #66 0x00005600a1969009 in _PyEval_EvalFrameDefault () #67 0x00005600a198661e in ?? () #68 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #69 0x00005600a198661e in ?? () #70 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #71 0x00005600a196e574 in _PyObject_FastCallDictTstate () #72 0x00005600a198362c in _PyObject_Call_Prepend () #73 0x00005600a1a8b464 in ?? () #74 0x00005600a196f39b in _PyObject_MakeTpCall () #75 0x00005600a1969009 in _PyEval_EvalFrameDefault () #76 0x00005600a19864d1 in ?? () #77 0x00005600a1968702 in _PyEval_EvalFrameDefault () #78 0x00005600a19864d1 in ?? () #79 0x00005600a1987172 in PyObject_Call () #80 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #81 0x00005600a19864d1 in ?? () #82 0x00005600a1987172 in PyObject_Call () #83 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #84 0x00005600a197938c in _PyFunction_Vectorcall () #85 0x00005600a196e61d in _PyObject_FastCallDictTstate () #86 0x00005600a198362c in _PyObject_Call_Prepend () #87 0x00005600a1a8b464 in ?? () #88 0x00005600a198710b in PyObject_Call () #89 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #90 0x00005600a197938c in _PyFunction_Vectorcall () #91 0x00005600a196397f in _PyEval_EvalFrameDefault () #92 0x00005600a197938c in _PyFunction_Vectorcall () #93 0x00005600a196e61d in _PyObject_FastCallDictTstate () #94 0x00005600a198362c in _PyObject_Call_Prepend () #95 0x00005600a1a8b464 in ?? () #96 0x00005600a196f39b in _PyObject_MakeTpCall () #97 0x00005600a196999e in _PyEval_EvalFrameDefault () #98 0x00005600a19864d1 in ?? () #99 0x00005600a19649ab in _PyEval_EvalFrameDefault () #100 0x00005600a197938c in _PyFunction_Vectorcall () #101 0x00005600a196397f in _PyEval_EvalFrameDefault () #102 0x00005600a197938c in _PyFunction_Vectorcall () #103 0x00005600a1963790 in _PyEval_EvalFrameDefault () #104 0x00005600a197938c in _PyFunction_Vectorcall () #105 0x00005600a1963790 in _PyEval_EvalFrameDefault () #106 0x00005600a197938c in _PyFunction_Vectorcall () #107 0x00005600a1963790 in _PyEval_EvalFrameDefault () #108 0x00005600a197938c in _PyFunction_Vectorcall () #109 0x00005600a1963790 in _PyEval_EvalFrameDefault () #110 0x00005600a1a48a16 in ?? () #111 0x00005600a1a488e6 in PyEval_EvalCode () #112 0x00005600a1a6f108 in ?? () #113 0x00005600a1a6990f in ?? () #114 0x00005600a1a6eea5 in ?? () #115 0x00005600a1a6e488 in _PyRun_SimpleFileObject () #116 0x00005600a1a6e167 in _PyRun_AnyFileObject () #117 0x00005600a1a628ae in Py_RunMain () #118 0x00005600a1a3c93d in Py_BytesMain () #119 0x00007f3913dfed90 in __libc_start_call_main (main=main entry=0x5600a1a3c900, argc=argc entry=10, argv=argv entry=0x7ffd77ff6528) at ../sysdeps/nptl/libc_start_call_main.h:58 #120 0x00007f3913dfee40 in __libc_start_main_impl (main=0x5600a1a3c900, argc=10, argv=0x7ffd77ff6528, init=, fini=, rtld_fini=, stack_end=0x7ffd77ff6518) at ../csu/libc-start.c:392 #121 0x00005600a1a3c835 in _start () =========================================================== *** Break *** segmentation violation =========================================================== There was a crash. This is the entire stack trace of all threads: =========================================================== Thread 59 (Thread 0x7f37664bb640 (LWP 954713) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 58 (Thread 0x7f3766cbc640 (LWP 954712) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 57 (Thread 0x7f37674bd640 (LWP 954711) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 56 (Thread 0x7f3767cbe640 (LWP 954710) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 55 (Thread 0x7f37684bf640 (LWP 954709) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 54 (Thread 0x7f3768cc0640 (LWP 954708) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 53 (Thread 0x7f37694c1640 (LWP 954707) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 52 (Thread 0x7f3769cc2640 (LWP 954706) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 51 (Thread 0x7f376a4c3640 (LWP 954705) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 50 (Thread 0x7f376acc4640 (LWP 954704) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 49 (Thread 0x7f376b4c5640 (LWP 954703) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 48 (Thread 0x7f376bcc6640 (LWP 954702) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 47 (Thread 0x7f376c4c7640 (LWP 954701) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 46 (Thread 0x7f376ccc8640 (LWP 954700) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 45 (Thread 0x7f376d4c9640 (LWP 954699) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 44 (Thread 0x7f376dcca640 (LWP 954698) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 43 (Thread 0x7f376e4cb640 (LWP 954697) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 42 (Thread 0x7f376eccc640 (LWP 954696) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 41 (Thread 0x7f376f4cd640 (LWP 954695) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 40 (Thread 0x7f376fcce640 (LWP 954694) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 39 (Thread 0x7f37704cf640 (LWP 954693) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 38 (Thread 0x7f3770cd0640 (LWP 954692) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 37 (Thread 0x7f37714d1640 (LWP 954691) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 36 (Thread 0x7f3771cd2640 (LWP 954690) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 35 (Thread 0x7f37724d3640 (LWP 954689) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 34 (Thread 0x7f3772cd4640 (LWP 954688) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 33 (Thread 0x7f37734d5640 (LWP 954687) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 32 (Thread 0x7f3773cd6640 (LWP 954686) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 31 (Thread 0x7f37744d7640 (LWP 954685) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 30 (Thread 0x7f3774cd8640 (LWP 954684) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 29 (Thread 0x7f37754d9640 (LWP 954683) "python3"): #0 0x00007f3912e18b31 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #1 0x00007f3912e163e0 in ?? () from /usr/local/lib/python3.10/dist-packages/torch/lib/libgomp-a34b3233.so.1 #2 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #3 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 28 (Thread 0x7f3797fff640 (LWP 954671) "python3"): #0 __futex_abstimed_wait_common64 (private=, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3790000e00) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3790000e00) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3790000e00, expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=) at ./nptl/futex-internal.c:139 #3 0x00007f3913e71bdf in do_futex_wait (sem=sem entry=0x7f3790000e00, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111 #4 0x00007f3913e71c78 in __new_sem_wait_slow64 (sem=0x7f3790000e00, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183 #5 0x00005600a193add0 in PyThread_acquire_lock_timed () #6 0x00005600a198ea64 in ?? () #7 0x00005600a1979de7 in ?? () #8 0x00005600a196397f in _PyEval_EvalFrameDefault () #9 0x00005600a19864d1 in ?? () #10 0x00005600a1963790 in _PyEval_EvalFrameDefault () #11 0x00005600a197938c in _PyFunction_Vectorcall () #12 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #13 0x00005600a197938c in _PyFunction_Vectorcall () #14 0x00005600a196397f in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00005600a196397f in _PyEval_EvalFrameDefault () #17 0x00005600a1986731 in ?? () #18 0x00005600a1a9b7ca in ?? () #19 0x00005600a1a91138 in ?? () #20 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #21 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 27 (Thread 0x7f37b4ffd640 (LWP 954669) "python3"): #0 __futex_abstimed_wait_common64 (private=, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f379c001020) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=, abstime=0x0, clockid=0, expected=0, futex_word=0x7f379c001020) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f379c001020, expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=) at ./nptl/futex-internal.c:139 #3 0x00007f3913e71bdf in do_futex_wait (sem=sem entry=0x7f379c001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111 #4 0x00007f3913e71c78 in __new_sem_wait_slow64 (sem=0x7f379c001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183 #5 0x00005600a193add0 in PyThread_acquire_lock_timed () #6 0x00005600a198ea64 in ?? () #7 0x00005600a1979de7 in ?? () #8 0x00005600a196397f in _PyEval_EvalFrameDefault () #9 0x00005600a19864d1 in ?? () #10 0x00005600a1963790 in _PyEval_EvalFrameDefault () #11 0x00005600a197938c in _PyFunction_Vectorcall () #12 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #13 0x00005600a197938c in _PyFunction_Vectorcall () #14 0x00005600a196397f in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00005600a196397f in _PyEval_EvalFrameDefault () #17 0x00005600a1986731 in ?? () #18 0x00005600a1a9b7ca in ?? () #19 0x00005600a1a91138 in ?? () #20 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #21 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 26 (Thread 0x7f37b57fe640 (LWP 954667) "python3"): #0 __futex_abstimed_wait_common64 (private=, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3798001020) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3798001020) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3798001020, expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=) at ./nptl/futex-internal.c:139 #3 0x00007f3913e71bdf in do_futex_wait (sem=sem entry=0x7f3798001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111 #4 0x00007f3913e71c78 in __new_sem_wait_slow64 (sem=0x7f3798001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183 #5 0x00005600a193add0 in PyThread_acquire_lock_timed () #6 0x00005600a198ea64 in ?? () #7 0x00005600a1979de7 in ?? () #8 0x00005600a196397f in _PyEval_EvalFrameDefault () #9 0x00005600a19864d1 in ?? () #10 0x00005600a1963790 in _PyEval_EvalFrameDefault () #11 0x00005600a197938c in _PyFunction_Vectorcall () #12 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #13 0x00005600a197938c in _PyFunction_Vectorcall () #14 0x00005600a196397f in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00005600a196397f in _PyEval_EvalFrameDefault () #17 0x00005600a1986731 in ?? () #18 0x00005600a1a9b7ca in ?? () #19 0x00005600a1a91138 in ?? () #20 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #21 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 25 (Thread 0x7f37b5fff640 (LWP 954666) "python3"): #0 __futex_abstimed_wait_common64 (private=, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f37a4001020) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=, abstime=0x0, clockid=0, expected=0, futex_word=0x7f37a4001020) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f37a4001020, expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=) at ./nptl/futex-internal.c:139 #3 0x00007f3913e71bdf in do_futex_wait (sem=sem entry=0x7f37a4001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111 #4 0x00007f3913e71c78 in __new_sem_wait_slow64 (sem=0x7f37a4001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183 #5 0x00005600a193add0 in PyThread_acquire_lock_timed () #6 0x00005600a198ea64 in ?? () #7 0x00005600a1979de7 in ?? () #8 0x00005600a196397f in _PyEval_EvalFrameDefault () #9 0x00005600a19864d1 in ?? () #10 0x00005600a1963790 in _PyEval_EvalFrameDefault () #11 0x00005600a197938c in _PyFunction_Vectorcall () #12 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #13 0x00005600a197938c in _PyFunction_Vectorcall () #14 0x00005600a196397f in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00005600a196397f in _PyEval_EvalFrameDefault () #17 0x00005600a1986731 in ?? () #18 0x00005600a1a9b7ca in ?? () #19 0x00005600a1a91138 in ?? () #20 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #21 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 24 (Thread 0x7f37e4a10640 (LWP 954665) "python3"): #0 __futex_abstimed_wait_common64 (private=, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f37a0001020) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=, abstime=0x0, clockid=0, expected=0, futex_word=0x7f37a0001020) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f37a0001020, expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=) at ./nptl/futex-internal.c:139 #3 0x00007f3913e71bdf in do_futex_wait (sem=sem entry=0x7f37a0001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111 #4 0x00007f3913e71c78 in __new_sem_wait_slow64 (sem=0x7f37a0001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183 #5 0x00005600a193add0 in PyThread_acquire_lock_timed () #6 0x00005600a198ea64 in ?? () #7 0x00005600a1979de7 in ?? () #8 0x00005600a196397f in _PyEval_EvalFrameDefault () #9 0x00005600a19864d1 in ?? () #10 0x00005600a1963790 in _PyEval_EvalFrameDefault () #11 0x00005600a197938c in _PyFunction_Vectorcall () #12 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #13 0x00005600a197938c in _PyFunction_Vectorcall () #14 0x00005600a196397f in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00005600a196397f in _PyEval_EvalFrameDefault () #17 0x00005600a1986731 in ?? () #18 0x00005600a1a9b7ca in ?? () #19 0x00005600a1a91138 in ?? () #20 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #21 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 23 (Thread 0x7f37e5211640 (LWP 954664) "python3"): #0 __futex_abstimed_wait_common64 (private=, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f37ac001020) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=, abstime=0x0, clockid=0, expected=0, futex_word=0x7f37ac001020) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f37ac001020, expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=) at ./nptl/futex-internal.c:139 #3 0x00007f3913e71bdf in do_futex_wait (sem=sem entry=0x7f37ac001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111 #4 0x00007f3913e71c78 in __new_sem_wait_slow64 (sem=0x7f37ac001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183 #5 0x00005600a193add0 in PyThread_acquire_lock_timed () #6 0x00005600a198ea64 in ?? () #7 0x00005600a1979de7 in ?? () #8 0x00005600a196397f in _PyEval_EvalFrameDefault () #9 0x00005600a19864d1 in ?? () #10 0x00005600a1963790 in _PyEval_EvalFrameDefault () #11 0x00005600a197938c in _PyFunction_Vectorcall () #12 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #13 0x00005600a197938c in _PyFunction_Vectorcall () #14 0x00005600a196397f in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00005600a196397f in _PyEval_EvalFrameDefault () #17 0x00005600a1986731 in ?? () #18 0x00005600a1a9b7ca in ?? () #19 0x00005600a1a91138 in ?? () #20 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #21 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 22 (Thread 0x7f37e5a12640 (LWP 954663) "python3"): #0 __futex_abstimed_wait_common64 (private=, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f37a8001020) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=, abstime=0x0, clockid=0, expected=0, futex_word=0x7f37a8001020) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f37a8001020, expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=) at ./nptl/futex-internal.c:139 #3 0x00007f3913e71bdf in do_futex_wait (sem=sem entry=0x7f37a8001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111 #4 0x00007f3913e71c78 in __new_sem_wait_slow64 (sem=0x7f37a8001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183 #5 0x00005600a193add0 in PyThread_acquire_lock_timed () #6 0x00005600a198ea64 in ?? () #7 0x00005600a1979de7 in ?? () #8 0x00005600a196397f in _PyEval_EvalFrameDefault () #9 0x00005600a19864d1 in ?? () #10 0x00005600a1963790 in _PyEval_EvalFrameDefault () #11 0x00005600a197938c in _PyFunction_Vectorcall () #12 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #13 0x00005600a197938c in _PyFunction_Vectorcall () #14 0x00005600a196397f in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00005600a196397f in _PyEval_EvalFrameDefault () #17 0x00005600a1986731 in ?? () #18 0x00005600a1a9b7ca in ?? () #19 0x00005600a1a91138 in ?? () #20 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #21 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 21 (Thread 0x7f37e6213640 (LWP 954636) "python3"): #0 __futex_abstimed_wait_common64 (private=, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f37b0001020) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=, abstime=0x0, clockid=0, expected=0, futex_word=0x7f37b0001020) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f37b0001020, expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=) at ./nptl/futex-internal.c:139 #3 0x00007f3913e71bdf in do_futex_wait (sem=sem entry=0x7f37b0001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111 #4 0x00007f3913e71c78 in __new_sem_wait_slow64 (sem=0x7f37b0001020, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183 #5 0x00005600a193add0 in PyThread_acquire_lock_timed () #6 0x00005600a198ea64 in ?? () #7 0x00005600a1979de7 in ?? () #8 0x00005600a196397f in _PyEval_EvalFrameDefault () #9 0x00005600a19864d1 in ?? () #10 0x00005600a1963790 in _PyEval_EvalFrameDefault () #11 0x00005600a197938c in _PyFunction_Vectorcall () #12 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #13 0x00005600a197938c in _PyFunction_Vectorcall () #14 0x00005600a196397f in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00005600a196397f in _PyEval_EvalFrameDefault () #17 0x00005600a1986731 in ?? () #18 0x00005600a1a9b7ca in ?? () #19 0x00005600a1a91138 in ?? () #20 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #21 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 20 (Thread 0x7f37e7e55640 (LWP 952858) "cuda-EvtHandlr"): #0 0x00007f3913eedbcf in __GI___poll (fds=0x7f37e0000c80, nfds=13, timeout=100) at ../sysdeps/unix/sysv/linux/poll.c:29 #1 0x00007f3858aaa44f in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1 #2 0x00007f3858b6e80f in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1 #3 0x00007f3858aa2143 in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1 #4 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #5 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 19 (Thread 0x7f3810947640 (LWP 951379) "cuda0000340000e"): #0 0x00007f3913eedbcf in __GI___poll (fds=0x5600b177d990, nfds=3, timeout=-1) at ../sysdeps/unix/sysv/linux/poll.c:29 #1 0x00007f3858aaa44f in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1 #2 0x00007f3858b6e80f in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1 #3 0x00007f3858aa2143 in ?? () from /opt/udiImage/modules/gpu/lib64/libcuda.so.1 #4 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #5 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 18 (Thread 0x7f381ca57640 (LWP 951376) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 17 (Thread 0x7f381d258640 (LWP 951375) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 16 (Thread 0x7f381da59640 (LWP 951374) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 15 (Thread 0x7f381e25a640 (LWP 951373) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 14 (Thread 0x7f381ea5b640 (LWP 951372) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 13 (Thread 0x7f381f25c640 (LWP 951371) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 12 (Thread 0x7f381fa5d640 (LWP 951370) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 11 (Thread 0x7f382025e640 (LWP 951369) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 10 (Thread 0x7f3820a5f640 (LWP 951368) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 9 (Thread 0x7f3821260640 (LWP 951367) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 8 (Thread 0x7f3821a61640 (LWP 951366) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 7 (Thread 0x7f3822262640 (LWP 951365) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 6 (Thread 0x7f3822a63640 (LWP 951364) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 5 (Thread 0x7f3823264640 (LWP 951363) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 4 (Thread 0x7f3823a65640 (LWP 951362) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 3 (Thread 0x7f3824266640 (LWP 951361) "python3"): #0 __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x0, op=393, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:57 #1 __futex_abstimed_wait_common (cancel=true, private=0, abstime=0x0, clockid=0, expected=0, futex_word=0x7f3824394bb8 ) at ./nptl/futex-internal.c:87 #2 __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word entry=0x7f3824394bb8 , expected=expected entry=0, clockid=clockid entry=0, abstime=abstime entry=0x0, private=private entry=0) at ./nptl/futex-internal.c:139 #3 0x00007f3913e68a41 in __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x7f3824394b68 , cond=0x7f3824394b90 ) at ./nptl/pthread_cond_wait.c:503 #4 ___pthread_cond_wait (cond=0x7f3824394b90 , mutex=0x7f3824394b68 ) at ./nptl/pthread_cond_wait.c:627 #5 0x00007f382438ecef in th_worker (tidptr=) at numexpr/module.cpp:62 #6 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #7 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 2 (Thread 0x7f382fb62640 (LWP 951351) "python3"): #0 0x00007f3913ef063d in __GI___select (nfds=0, readfds=0x0, writefds=0x0, exceptfds=0x0, timeout=0x7f382fb615d0) at ../sysdeps/unix/sysv/linux/select.c:69 #1 0x00005600a1a832f6 in ?? () #2 0x00005600a1978534 in ?? () #3 0x00005600a1968702 in _PyEval_EvalFrameDefault () #4 0x00005600a197938c in _PyFunction_Vectorcall () #5 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #6 0x00005600a197938c in _PyFunction_Vectorcall () #7 0x00005600a196397f in _PyEval_EvalFrameDefault () #8 0x00005600a197938c in _PyFunction_Vectorcall () #9 0x00005600a196397f in _PyEval_EvalFrameDefault () #10 0x00005600a1986731 in ?? () #11 0x00005600a1a9b7ca in ?? () #12 0x00005600a1a91138 in ?? () #13 0x00007f3913e69ac3 in start_thread (arg=) at ./nptl/pthread_create.c:442 #14 0x00007f3913efb850 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 Thread 1 (Thread 0x7f3913dd0740 (LWP 951334) "python3"): #0 0x00007f3913ebf42f in __GI___wait4 (pid=964740, stat_loc=stat_loc entry=0x7ffd77fecd68, options=options entry=0, usage=usage entry=0x0) at ../sysdeps/unix/sysv/linux/wait4.c:30 #1 0x00007f3913ebf3ab in __GI___waitpid (pid=, stat_loc=stat_loc entry=0x7ffd77fecd68, options=options entry=0) at ./posix/waitpid.c:38 #2 0x00007f3913e25bdb in do_system (line=) at ../sysdeps/posix/system.c:171 #3 0x00007f3846e0fc84 in TUnixSystem::StackTrace() () from /app/root/lib/libCore.so #4 0x00007f38470b7e77 in (anonymous namespace)::TExceptionHandlerImp::HandleException(int) () from /app/root/lib/libcppyy_backend3_10.so #5 0x00007f3846e0ce19 in TUnixSystem::DispatchSignals(ESignals) () from /app/root/lib/libCore.so #6 #7 0x00007f382cb5f9f6 in minkowski::CoordinateMapGPU::stride(std::vector > const&) const () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #8 0x00007f382cb9b559 in minkowski::CoordinateMapManager::stride(std::pair >, std::string> const&, std::vector > const&, std::string) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #9 0x00007f382cb2207a in at::Tensor minkowski::ConvolutionForwardGPU(at::Tensor const&, at::Tensor const&, std::vector > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #10 0x00007f382cc8b2d1 in pybind11::cpp_function::initialize > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*), at::Tensor, at::Tensor const&, at::Tensor const&, std::vector > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*, pybind11::name, pybind11::scope, pybind11::sibling, pybind11::call_guard >(at::Tensor (*&)(at::Tensor const&, at::Tensor const&, std::vector > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*), at::Tensor (*)(at::Tensor const&, at::Tensor const&, std::vector > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*), pybind11::name const&, pybind11::scope const&, pybind11::sibling const&, pybind11::call_guard const&)::{lambda(pybind11::detail::function_call&)#3}::_FUN(pybind11::detail::function_call&) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #11 0x00007f382cba6567 in pybind11::cpp_function::dispatcher(_object*, _object*, _object*) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #12 0x00005600a1978b32 in ?? () #13 0x00005600a196f39b in _PyObject_MakeTpCall () #14 0x00005600a1968a97 in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00007f3912244249 in THPFunction_apply(_object*, _object*) () from /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_python.so #17 0x00005600a1978b58 in ?? () #18 0x00005600a198710b in PyObject_Call () #19 0x00005600a19696ef in _PyEval_EvalFrameDefault () #20 0x00005600a19864d1 in ?? () #21 0x00005600a1968702 in _PyEval_EvalFrameDefault () #22 0x00005600a198661e in ?? () #23 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #24 0x00005600a198661e in ?? () #25 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #26 0x00005600a196e574 in _PyObject_FastCallDictTstate () #27 0x00005600a198362c in _PyObject_Call_Prepend () #28 0x00005600a1a8b464 in ?? () #29 0x00005600a196f39b in _PyObject_MakeTpCall () #30 0x00005600a1968a97 in _PyEval_EvalFrameDefault () #31 0x00005600a198661e in ?? () #32 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #33 0x00005600a198661e in ?? () #34 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #35 0x00005600a196e574 in _PyObject_FastCallDictTstate () #36 0x00005600a198362c in _PyObject_Call_Prepend () #37 0x00005600a1a8b464 in ?? () #38 0x00005600a196f39b in _PyObject_MakeTpCall () #39 0x00005600a1968a97 in _PyEval_EvalFrameDefault () #40 0x00005600a198661e in ?? () #41 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #42 0x00005600a198661e in ?? () #43 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #44 0x00005600a196e574 in _PyObject_FastCallDictTstate () #45 0x00005600a198362c in _PyObject_Call_Prepend () #46 0x00005600a1a8b464 in ?? () #47 0x00005600a196f39b in _PyObject_MakeTpCall () #48 0x00005600a1969009 in _PyEval_EvalFrameDefault () #49 0x00005600a198661e in ?? () #50 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #51 0x00005600a198661e in ?? () #52 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #53 0x00005600a196e574 in _PyObject_FastCallDictTstate () #54 0x00005600a198362c in _PyObject_Call_Prepend () #55 0x00005600a1a8b464 in ?? () #56 0x00005600a196f39b in _PyObject_MakeTpCall () #57 0x00005600a1969009 in _PyEval_EvalFrameDefault () #58 0x00005600a198661e in ?? () #59 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #60 0x00005600a198661e in ?? () #61 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #62 0x00005600a196e574 in _PyObject_FastCallDictTstate () #63 0x00005600a198362c in _PyObject_Call_Prepend () #64 0x00005600a1a8b464 in ?? () #65 0x00005600a196f39b in _PyObject_MakeTpCall () #66 0x00005600a1969009 in _PyEval_EvalFrameDefault () #67 0x00005600a198661e in ?? () #68 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #69 0x00005600a198661e in ?? () #70 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #71 0x00005600a196e574 in _PyObject_FastCallDictTstate () #72 0x00005600a198362c in _PyObject_Call_Prepend () #73 0x00005600a1a8b464 in ?? () #74 0x00005600a196f39b in _PyObject_MakeTpCall () #75 0x00005600a1969009 in _PyEval_EvalFrameDefault () #76 0x00005600a19864d1 in ?? () #77 0x00005600a1968702 in _PyEval_EvalFrameDefault () #78 0x00005600a19864d1 in ?? () #79 0x00005600a1987172 in PyObject_Call () #80 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #81 0x00005600a19864d1 in ?? () #82 0x00005600a1987172 in PyObject_Call () #83 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #84 0x00005600a197938c in _PyFunction_Vectorcall () #85 0x00005600a196e61d in _PyObject_FastCallDictTstate () #86 0x00005600a198362c in _PyObject_Call_Prepend () #87 0x00005600a1a8b464 in ?? () #88 0x00005600a198710b in PyObject_Call () #89 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #90 0x00005600a197938c in _PyFunction_Vectorcall () #91 0x00005600a196397f in _PyEval_EvalFrameDefault () #92 0x00005600a197938c in _PyFunction_Vectorcall () #93 0x00005600a196e61d in _PyObject_FastCallDictTstate () #94 0x00005600a198362c in _PyObject_Call_Prepend () #95 0x00005600a1a8b464 in ?? () #96 0x00005600a196f39b in _PyObject_MakeTpCall () #97 0x00005600a196999e in _PyEval_EvalFrameDefault () #98 0x00005600a19864d1 in ?? () #99 0x00005600a19649ab in _PyEval_EvalFrameDefault () #100 0x00005600a197938c in _PyFunction_Vectorcall () #101 0x00005600a196397f in _PyEval_EvalFrameDefault () #102 0x00005600a197938c in _PyFunction_Vectorcall () #103 0x00005600a1963790 in _PyEval_EvalFrameDefault () #104 0x00005600a197938c in _PyFunction_Vectorcall () #105 0x00005600a1963790 in _PyEval_EvalFrameDefault () #106 0x00005600a197938c in _PyFunction_Vectorcall () #107 0x00005600a1963790 in _PyEval_EvalFrameDefault () #108 0x00005600a197938c in _PyFunction_Vectorcall () #109 0x00005600a1963790 in _PyEval_EvalFrameDefault () #110 0x00005600a1a48a16 in ?? () #111 0x00005600a1a488e6 in PyEval_EvalCode () #112 0x00005600a1a6f108 in ?? () #113 0x00005600a1a6990f in ?? () #114 0x00005600a1a6eea5 in ?? () #115 0x00005600a1a6e488 in _PyRun_SimpleFileObject () #116 0x00005600a1a6e167 in _PyRun_AnyFileObject () #117 0x00005600a1a628ae in Py_RunMain () #118 0x00005600a1a3c93d in Py_BytesMain () #119 0x00007f3913dfed90 in __libc_start_call_main (main=main entry=0x5600a1a3c900, argc=argc entry=10, argv=argv entry=0x7ffd77ff6528) at ../sysdeps/nptl/libc_start_call_main.h:58 #120 0x00007f3913dfee40 in __libc_start_main_impl (main=0x5600a1a3c900, argc=10, argv=0x7ffd77ff6528, init=, fini=, rtld_fini=, stack_end=0x7ffd77ff6518) at ../csu/libc-start.c:392 #121 0x00005600a1a3c835 in _start () =========================================================== The lines below might hint at the cause of the crash. You may get help by asking at the ROOT forum https://root.cern/forum Only if you are really convinced it is a bug in ROOT then please submit a report at https://root.cern/bugs Please post the ENTIRE stack trace from above as an attachment in addition to anything else that might help us fixing this issue. =========================================================== #7 0x00007f382cb5f9f6 in minkowski::CoordinateMapGPU::stride(std::vector > const&) const () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #8 0x00007f382cb9b559 in minkowski::CoordinateMapManager::stride(std::pair >, std::string> const&, std::vector > const&, std::string) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #9 0x00007f382cb2207a in at::Tensor minkowski::ConvolutionForwardGPU(at::Tensor const&, at::Tensor const&, std::vector > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #10 0x00007f382cc8b2d1 in pybind11::cpp_function::initialize > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*), at::Tensor, at::Tensor const&, at::Tensor const&, std::vector > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*, pybind11::name, pybind11::scope, pybind11::sibling, pybind11::call_guard >(at::Tensor (*&)(at::Tensor const&, at::Tensor const&, std::vector > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*), at::Tensor (*)(at::Tensor const&, at::Tensor const&, std::vector > const&, std::vector > const&, std::vector > const&, minkowski::RegionType::Type, at::Tensor const&, bool, minkowski::ConvolutionMode::Type, minkowski::CoordinateMapKey*, minkowski::CoordinateMapKey*, minkowski::CoordinateMapManager*), pybind11::name const&, pybind11::scope const&, pybind11::sibling const&, pybind11::call_guard const&)::{lambda(pybind11::detail::function_call&)#3}::_FUN(pybind11::detail::function_call&) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #11 0x00007f382cba6567 in pybind11::cpp_function::dispatcher(_object*, _object*, _object*) () from /usr/local/lib/python3.10/dist-packages/MinkowskiEngineBackend/_C.cpython-310-x86_64-linux-gnu.so #12 0x00005600a1978b32 in ?? () #13 0x00005600a196f39b in _PyObject_MakeTpCall () #14 0x00005600a1968a97 in _PyEval_EvalFrameDefault () #15 0x00005600a197938c in _PyFunction_Vectorcall () #16 0x00007f3912244249 in THPFunction_apply(_object*, _object*) () from /usr/local/lib/python3.10/dist-packages/torch/lib/libtorch_python.so #17 0x00005600a1978b58 in ?? () #18 0x00005600a198710b in PyObject_Call () #19 0x00005600a19696ef in _PyEval_EvalFrameDefault () #20 0x00005600a19864d1 in ?? () #21 0x00005600a1968702 in _PyEval_EvalFrameDefault () #22 0x00005600a198661e in ?? () #23 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #24 0x00005600a198661e in ?? () #25 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #26 0x00005600a196e574 in _PyObject_FastCallDictTstate () #27 0x00005600a198362c in _PyObject_Call_Prepend () #28 0x00005600a1a8b464 in ?? () #29 0x00005600a196f39b in _PyObject_MakeTpCall () #30 0x00005600a1968a97 in _PyEval_EvalFrameDefault () #31 0x00005600a198661e in ?? () #32 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #33 0x00005600a198661e in ?? () #34 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #35 0x00005600a196e574 in _PyObject_FastCallDictTstate () #36 0x00005600a198362c in _PyObject_Call_Prepend () #37 0x00005600a1a8b464 in ?? () #38 0x00005600a196f39b in _PyObject_MakeTpCall () #39 0x00005600a1968a97 in _PyEval_EvalFrameDefault () #40 0x00005600a198661e in ?? () #41 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #42 0x00005600a198661e in ?? () #43 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #44 0x00005600a196e574 in _PyObject_FastCallDictTstate () #45 0x00005600a198362c in _PyObject_Call_Prepend () #46 0x00005600a1a8b464 in ?? () #47 0x00005600a196f39b in _PyObject_MakeTpCall () #48 0x00005600a1969009 in _PyEval_EvalFrameDefault () #49 0x00005600a198661e in ?? () #50 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #51 0x00005600a198661e in ?? () #52 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #53 0x00005600a196e574 in _PyObject_FastCallDictTstate () #54 0x00005600a198362c in _PyObject_Call_Prepend () #55 0x00005600a1a8b464 in ?? () #56 0x00005600a196f39b in _PyObject_MakeTpCall () #57 0x00005600a1969009 in _PyEval_EvalFrameDefault () #58 0x00005600a198661e in ?? () #59 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #60 0x00005600a198661e in ?? () #61 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #62 0x00005600a196e574 in _PyObject_FastCallDictTstate () #63 0x00005600a198362c in _PyObject_Call_Prepend () #64 0x00005600a1a8b464 in ?? () #65 0x00005600a196f39b in _PyObject_MakeTpCall () #66 0x00005600a1969009 in _PyEval_EvalFrameDefault () #67 0x00005600a198661e in ?? () #68 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #69 0x00005600a198661e in ?? () #70 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #71 0x00005600a196e574 in _PyObject_FastCallDictTstate () #72 0x00005600a198362c in _PyObject_Call_Prepend () #73 0x00005600a1a8b464 in ?? () #74 0x00005600a196f39b in _PyObject_MakeTpCall () #75 0x00005600a1969009 in _PyEval_EvalFrameDefault () #76 0x00005600a19864d1 in ?? () #77 0x00005600a1968702 in _PyEval_EvalFrameDefault () #78 0x00005600a19864d1 in ?? () #79 0x00005600a1987172 in PyObject_Call () #80 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #81 0x00005600a19864d1 in ?? () #82 0x00005600a1987172 in PyObject_Call () #83 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #84 0x00005600a197938c in _PyFunction_Vectorcall () #85 0x00005600a196e61d in _PyObject_FastCallDictTstate () #86 0x00005600a198362c in _PyObject_Call_Prepend () #87 0x00005600a1a8b464 in ?? () #88 0x00005600a198710b in PyObject_Call () #89 0x00005600a1965c30 in _PyEval_EvalFrameDefault () #90 0x00005600a197938c in _PyFunction_Vectorcall () #91 0x00005600a196397f in _PyEval_EvalFrameDefault () #92 0x00005600a197938c in _PyFunction_Vectorcall () #93 0x00005600a196e61d in _PyObject_FastCallDictTstate () #94 0x00005600a198362c in _PyObject_Call_Prepend () #95 0x00005600a1a8b464 in ?? () #96 0x00005600a196f39b in _PyObject_MakeTpCall () #97 0x00005600a196999e in _PyEval_EvalFrameDefault () #98 0x00005600a19864d1 in ?? () #99 0x00005600a19649ab in _PyEval_EvalFrameDefault () #100 0x00005600a197938c in _PyFunction_Vectorcall () #101 0x00005600a196397f in _PyEval_EvalFrameDefault () #102 0x00005600a197938c in _PyFunction_Vectorcall () #103 0x00005600a1963790 in _PyEval_EvalFrameDefault () #104 0x00005600a197938c in _PyFunction_Vectorcall () #105 0x00005600a1963790 in _PyEval_EvalFrameDefault () #106 0x00005600a197938c in _PyFunction_Vectorcall () #107 0x00005600a1963790 in _PyEval_EvalFrameDefault () #108 0x00005600a197938c in _PyFunction_Vectorcall () #109 0x00005600a1963790 in _PyEval_EvalFrameDefault () #110 0x00005600a1a48a16 in ?? () #111 0x00005600a1a488e6 in PyEval_EvalCode () #112 0x00005600a1a6f108 in ?? () #113 0x00005600a1a6990f in ?? () #114 0x00005600a1a6eea5 in ?? () #115 0x00005600a1a6e488 in _PyRun_SimpleFileObject () #116 0x00005600a1a6e167 in _PyRun_AnyFileObject () #117 0x00005600a1a628ae in Py_RunMain () #118 0x00005600a1a3c93d in Py_BytesMain () #119 0x00007f3913dfed90 in __libc_start_call_main (main=main entry=0x5600a1a3c900, argc=argc entry=10, argv=argv entry=0x7ffd77ff6528) at ../sysdeps/nptl/libc_start_call_main.h:58 #120 0x00007f3913dfee40 in __libc_start_main_impl (main=0x5600a1a3c900, argc=10, argv=0x7ffd77ff6528, init=, fini=, rtld_fini=, stack_end=0x7ffd77ff6518) at ../csu/libc-start.c:392 #121 0x00005600a1a3c835 in _start () =========================================================== Exception in thread Thread-1 (_process_root_events): Traceback (most recent call last): File "/usr/lib/python3.10/threading.py", line 1016, in _bootstrap_inner self.run() File "/usr/lib/python3.10/threading.py", line 953, in run self._target(*self._args, **self._kwargs) File "/app/root/lib/ROOT/_application.py", line 91, in _process_root_events gSystem.ProcessEvents() ReferenceError: attempt to access a null-pointer