StrumpackSparseSolverMPIDist.hpp

Go to the documentation of this file.

/*
 * STRUMPACK -- STRUctured Matrices PACKage, Copyright (c) 2014, The
 * Regents of the University of California, through Lawrence Berkeley
 * National Laboratory (subject to receipt of any required approvals
 * from the U.S. Dept. of Energy).  All rights reserved.
 *
 * If you have questions about your rights to use or distribute this
 * software, please contact Berkeley Lab's Technology Transfer
 * Department at TTD@lbl.gov.
 *
 * NOTICE. This software is owned by the U.S. Department of Energy. As
 * such, the U.S. Government has been granted for itself and others
 * acting on its behalf a paid-up, nonexclusive, irrevocable,
 * worldwide license in the Software to reproduce, prepare derivative
 * works, and perform publicly and display publicly.  Beginning five
 * (5) years after the date permission to assert copyright is obtained
 * from the U.S. Department of Energy, and subject to any subsequent
 * five (5) year renewals, the U.S. Government is granted for itself
 * and others acting on its behalf a paid-up, nonexclusive,
 * irrevocable, worldwide license in the Software to reproduce,
 * prepare derivative works, distribute copies to the public, perform
 * publicly and display publicly, and to permit others to do so.
 *
 * Developers: Pieter Ghysels, Francois-Henry Rouet, Xiaoye S. Li.
 *             (Lawrence Berkeley National Lab, Computational Research
 *             Division).
 */
#ifndef STRUMPACK_SPARSE_SOLVER_MPI_DIST_HPP
#define STRUMPACK_SPARSE_SOLVER_MPI_DIST_HPP
 
#include "SparseSolverBase.hpp"
#include "dense/ScaLAPACKWrapper.hpp"
#include "dense/DistributedVector.hpp"
#include "sparse/CSRMatrixMPI.hpp"
 
namespace strumpack {
 
  // forward declarations
  template<typename scalar_t,typename integer_t> class EliminationTreeMPIDist;
  template<typename scalar_t,typename integer_t> class MatrixReorderingMPI;
 
  template<typename scalar_t,typename integer_t>
  class SparseSolverMPIDist :
    public SparseSolverBase<scalar_t,integer_t> {
 
    using SpMat_t = CompressedSparseMatrix<scalar_t,integer_t>;
    using Tree_t = EliminationTree<scalar_t,integer_t>;
    using Reord_t = MatrixReordering<scalar_t,integer_t>;
    using DenseM_t = DenseMatrix<scalar_t>;
    using DenseMW_t = DenseMatrixWrapper<scalar_t>;
 
  public:
    SparseSolverMPIDist(MPI_Comm comm, int argc, char* argv[],
                        bool verbose=true);
 
    SparseSolverMPIDist(MPI_Comm comm, bool verbose=true);
 
    ~SparseSolverMPIDist();
 
    void broadcast_matrix(const CSRMatrix<scalar_t,integer_t>& A);
 
    void broadcast_csr_matrix
    (integer_t N, const integer_t* row_ptr, const integer_t* col_ind,
     const scalar_t* values, bool symmetric_pattern=false);
 
    void set_matrix(const CSRMatrixMPI<scalar_t,integer_t>& A);
 
    void set_distributed_csr_matrix
    (integer_t local_rows, const integer_t* row_ptr,
     const integer_t* col_ind, const scalar_t* values,
     const integer_t* dist, bool symmetric_pattern=false);
 
    void set_MPIAIJ_matrix
    (integer_t local_rows,
     const integer_t* d_ptr, const integer_t* d_ind, const scalar_t* d_val,
     const integer_t* o_ptr, const integer_t* o_ind, const scalar_t* o_val,
     const integer_t* garray);
 
 
    void update_matrix_values
    (integer_t local_rows, const integer_t* row_ptr,
     const integer_t* col_ind, const scalar_t* values,
     const integer_t* dist, bool symmetric_pattern=false);
 
    void update_matrix_values(const CSRMatrixMPI<scalar_t,integer_t>& A);
 
    void update_MPIAIJ_matrix_values
    (integer_t local_rows, const integer_t* d_ptr, const integer_t* d_ind,
     const scalar_t* d_val, const integer_t* o_ptr, const integer_t* o_ind,
     const scalar_t* o_val, const integer_t* garray);
 
    MPI_Comm comm() const;
 
    const MPIComm& Comm() const { return comm_; }
 
  private:
    using SparseSolverBase<scalar_t,integer_t>::is_root_;
    using SparseSolverBase<scalar_t,integer_t>::opts_;
    MPIComm comm_;
 
    SpMat_t* matrix() override { return mat_mpi_.get(); }
    std::unique_ptr<SpMat_t> matrix_nonzero_diag() override {
      using real_t = typename RealType<scalar_t>::value_type;
      return mat_mpi_->add_missing_diagonal
        (std::sqrt(blas::lamch<real_t>('E')) * mat_mpi_->norm1());
    }
    Reord_t* reordering() override;
    Tree_t* tree() override { return tree_mpi_dist_.get(); }
    const SpMat_t* matrix() const override { return mat_mpi_.get(); }
    const Reord_t* reordering() const override;
    const Tree_t* tree() const override { return tree_mpi_dist_.get(); }
 
    void setup_tree() override;
    void setup_reordering() override;
    int compute_reordering(const int* p, int base, int nx, int ny, int nz,
                           int components, int width) override;
    void separator_reordering() override;
 
    void perf_counters_stop(const std::string& s) override;
    void synchronize() override { comm_.barrier(); }
    void reduce_flop_counters() const override;
 
    double max_peak_memory() const override {
      return comm_.reduce(double(params::peak_memory), MPI_MAX);
    }
    double min_peak_memory() const override {
      return comm_.reduce(double(params::peak_memory), MPI_MIN);
    }
 
    void redistribute_values();
 
    void delete_factors_internal() override;
 
    ReturnCode
    solve_internal(const scalar_t* b, scalar_t* x,
                   bool use_initial_guess=false) override;
    ReturnCode
    solve_internal(const DenseM_t& b, DenseM_t& x,
                   bool use_initial_guess=false) override;
    ReturnCode
    solve_internal(int nrhs, const scalar_t* b, int ldb,
                   scalar_t* x, int ldx,
                   bool use_initial_guess=false) override;
 
    std::unique_ptr<CSRMatrixMPI<scalar_t,integer_t>> mat_mpi_;
    std::unique_ptr<MatrixReorderingMPI<scalar_t,integer_t>> nd_mpi_;
    std::unique_ptr<EliminationTreeMPIDist<scalar_t,integer_t>> tree_mpi_dist_;
  };
 
  template<typename scalar_t,typename integer_t>
  using StrumpackSparseSolverMPIDist =
    SparseSolverMPIDist<scalar_t,integer_t>;
 
} // end namespace strumpack
 
#endif // STRUMPACK_SPARSE_SOLVER_MPI_DIST_HPP