B H])zModel._traces..rzBTime spent | Time remain.| Progress | {} | Traces/secTracerz%{} | {} | {} | {}/{} | {:,.2f}  )endg?)r3rtimer _verbositylenstrprintformatljustrange_print_refresh_ratedays_hours_mins_secs_str progress_barrjustsysstdoutflushnextrPRIORlog_importance_weightaddfinalize)r num_tracesr#r$r%r&map_funcsilentr'r5r*r/r0 generatorZtraces time_startlen_str_num_traces prev_durationidurationtraces_per_secondr2 log_weightrrr_traces%s4    J  z Model._tracescOst|j||S)N)rKr3)rr/r0rrr get_traceBszModel.get_tracec OsL|j||tj||||d|}|d|j|jd|t||d|S)N)rPr#r$rQr5r*zPrior, traces: {:,}prior)oprPr$r*)r[rrLrenamerAlength add_metadatar?) rrPr$rQr5r*r/r0r]rrrr]Es z Model.priorcCs|jS)N)r1)r2rrrr6Kr7zModel.cOs|j||||||d|S)N)rPr$rQr5r*)r])rrPr$rQr5r*r/r0rrr prior_resultsKszModel.prior_resultsc Os|tjkrb|j| |tj|d||||d| } | d| j| j| j d|t || j|dn|tj kr|j dkr~t dt,|j| |tj||j ||||d| } WdQRX| d| j|j j| j| j d|t || j||j jdnt|d} |dkrd d }|dkr>t|j| tj||d | } n|} t} d }d }d }|dkrdd }tjd krtt |}tdd|dd d }xt|D]}tjd kr~t| }||tjks||d kr~|}|d |}tdt|t|||t|d |t |d ||dd||d ddd|td |d|ddt j!"t|j| tj|| |d| }t#$| j%t#$|j%|j&| j&}xH|j'D]>}|j(r|t)|j*7}|t)| j+|j,j*8}|d 7}qW||j%7}t-j.dkr8tt/dddgdn|t)t-j.7}t#$t00t1|krl|d 7}|} ||d kr| 2|| qWtjd krt| 3| d|tj4krdnd| j|d krdnd |d||d||| j d|t ||||||d!| S)"N)rPr#r%r&rQr'r5r*z)Posterior, IS, traces: {:,}, ESS: {:,.2f} posterior)r^rPr%effective_sample_sizer*zCannot run inference engine IMPORTANCE_SAMPLING_WITH_INFERENCE_NETWORK because no inference network for this model is available. Use learn_inference_network or load_inference_network first.z>Posterior, IC, traces: {:,}, train. traces: {:,}, ESS: {:,.2f})r^rPr%rdr*Z train_traces)r5cSs|S)Nr)r2rrrr6]r7z!Model.posterior..)r#r%r'rrzVTime spent | Time remain.| Progress | {} | Accepted|Smp reuse| Traces/secr8r9z/{} | {} | {} | {}/{} | {} | {} | {:,.2f} z{:,.2f}%dr:)r;)r#r%r(r'zzWarning: trace did not hit the Metropolis Hastings site, ensure that the model is deterministic except pyprob.sample callsredbold)attrszIPosterior, {}, traces: {:,}{}, accepted: {:,.2f}%, sample reuse: {:,.2f}%ZLMHRMHz (thinning steps: {:,}))r^rPr%r*thinning_stepsZnum_traces_acceptedZnum_samples_reuised num_samples)5r IMPORTANCE_SAMPLINGr[r POSTERIORr_rAr`rdrar?*IMPORTANCE_SAMPLING_WITH_INFERENCE_NETWORKr RuntimeErrortorchno_grad_total_train_tracesrrKr3r<rr=r>r@rBrCrDrErFrGmaxrHrIrJmathloglength_controlledlog_prob_observedvariables_controlledreusedsumlog_probvariables_dict_addressaddressr-_metropolis_hastings_site_transition_log_probrrandomfloatrNrOLIGHTWEIGHT_METROPOLIS_HASTINGS)rrPr% initial_tracerQr'r5rlr*r/r0rcZ current_tracerTZtraces_acceptedZsamples_reusedZ samples_allrUrVrWrXrYZcandidate_traceZlog_acceptance_ratiovariablerrrrcNsr $   0$            $    NzModel.posteriorcCs|jS)N)r1)r2rrrr6r7c Os |j||||||||d| S)N)rPr%rrQr'r5rl)rc) rrPr%rrQr'r5rlr/r0rrrposterior_resultsszModel.posterior_resultscCs d|_dS)N)r)rrrrreset_inference_networkszModel.reset_inference_networkgeA@gMbP?gư>g?giXTi'rircCs0|dkrt||d}n t|d}|dkr.d}n t|d}|jdkrtd|tjkrft|||d|_n.|tjkrt|||||d|_nt d ||r|dk r|jj ||d|dk r|jj ||dntdtd |jj d |jj tjd |jj|||||| | | | |||||||||||d dS) N)modelr$) dataset_dirz!Creating new inference network...)robserve_embeddingsproposal_mixture_components)rrlstm_dim lstm_depthrzUnknown inference_network: {})save_file_name_prefixz1Continuing to train existing inference network...z Total number of parameters: {:,})device)rPdataset dataset_validnum_traces_end batch_size valid_everyoptimizer_typelearning_rate_initlearning_rate_endlearning_rate_scheduler_typemomentum weight_decayrsave_every_secdistributed_backend"distributed_params_sync_every_iterdistributed_num_bucketsdataloader_offline_num_workersstop_with_bad_loss log_file_name)rrrr@r FEEDFORWARDrLSTMr ValueErrorrAZ_pre_generate_layers_history_num_paramstor_deviceoptimize)rrPrr&r$rdataset_valid_dirrr valid_sizerrrrrrrrrZpre_generate_layersrrrrrrrrrrrrrrlearn_inference_networks,     zModel.learn_inference_networkcCs"|jdkrtd|j|dS)Nz+The model has no trained inference network.)rrq_save)rr5rrrsave_inference_networks zModel.save_inference_networkcCst||_||j_dS)N)InferenceNetworkBase_loadrZ_model)rr5rrrload_inference_networks zModel.load_inference_networkcOsNtj|s$td|t|t|d|d}|j||||d|dS)Nz&Directory does not exist, creating: {})r$)rrPnum_traces_per_file)ospathexistsr@rAmakedirsr save_dataset)rrrPrr$r/r0rrrrrs   zModel.save_dataset)rN) __name__ __module__ __qualname__rr!rrLrDISABLEDr rnr IW0r3r[r\r]rbrcrrr rr ADAMr NONErrrr __classcell__rr)rrrs "$FHrcs2eZdZd fdd ZfddZddZZS) RemoteModeltcp://127.0.0.1:5555Ncs*||_d|_||_||_tj||dS)N)_server_address _model_server_before_forward_func_after_forward_funcrr)rZserver_addressZbefore_forward_funcZafter_forward_funcr/r0)rrrrs zRemoteModel.__init__cs"|jdk r|jtdS)N)rcloser)r)rrrrs  zRemoteModel.closecCs^|jdkr,t|j|_d|jj|jj|_|jdk r>||j}|j dk rZ| |S)Nz{} running on {}) rrrrAZ model_nameZ system_namerrr!r)rretrrrr!s     zRemoteModel.forward)rNN)rrrrrr!rrr)rrrs r)rrr<rHrrvr termcolorr distributionsrrkrrrrr r r r r rnnrrrrrZremoterrrrrrrs  0  <