QCDistMDSystem.hpp

Go to the documentation of this file.
00001 //*****************************************************************************//
00002 //                                                                             //
00003 //   Copyright (c) 2001                                                        //
00004 //      INRIA                                                                  //
00005 //      54600 VILLERS LES NANCY                                                //
00006 //      France                                                                 //
00007 //                                                                             //
00008 //*****************************************************************************//
00009 //                                                                             //
00010 //               *** NOTICE OF PROPRIETARY INFORMATION ***                     //
00011 //                                                                             //
00012 // The information contained in this file is considered proprietary and the    //
00013 // exclusive property of  INRIA. This information may not be disclosed,        //
00014 // duplicated or used, in whole or in part, for  any purpose  whatsoever       //
00015 // without express written authorization from INRIA                            //
00016 //                                                                             //
00017 //*****************************************************************************//
00018 
00019 
00020 #ifndef QC_DIST_MD_SYSTEM_H_
00021 #define QC_DIST_MD_SYSTEM_H_
00022 
00023 
00024 #include <vector>
00025 #include <map>
00026 
00027 #include "QCCommon.hpp"
00028 #include "QCMDSystem.hpp"
00029 
00030 
00031 #if defined(HAVE_MPI) && defined(WITH_MPI_SUPPORT)
00032 
00033 #  include <mpi.h>
00034 
00035 #  define   HCONTRIB_ATOM           0
00036 #  define   HCONTRIB_ORDER          1
00037 #  define   HCONTRIB_OVRLP          2
00038 #  define   HCONTRIB_ZONE           3
00039 #  define   HCONTRIB_DENS           4
00040 
00041 
00042 #  define   FCONTRIB_ATOM           (0 + HCONTRIB_DENS) 
00043 #  define   FCONTRIB_ORDER          (1 + HCONTRIB_DENS)
00044 #  define   FCONTRIB_OVRLP          (2 + HCONTRIB_DENS)
00045 #  define   FCONTRIB_ZONE           (3 + HCONTRIB_DENS)
00046 #  define   FCONTRIB_BLOCK          (4 + HCONTRIB_DENS)
00047 #  define   FCONTRIB_DENS           (5 + HCONTRIB_DENS)
00048 
00049 
00050 #  define   DCONTRIB_INFO           (0 + FCONTRIB_DENS)
00051 #  define   DCONTRIB_ORDER          (1 + FCONTRIB_DENS)
00052 #  define   DCONTRIB_OVRLP          (2 + FCONTRIB_DENS)
00053 #  define   DCONTRIB_DENS           (3 + FCONTRIB_DENS)
00054 
00055 
00056 #  define   SDS_HEAD                1
00057 #  define   NB_SDS                  0
00058 
00059 
00060 
00064 struct QCFloatInt {
00065   
00066   QCFloat floatval;
00067   
00068   int     intval;
00069 };
00070 
00075 template <class TPSolver>
00076 class QCDistMDSystem :  public QCMDSystem<TPSolver>
00077 {
00078 
00080 public:
00084   typedef typename QCMDSystem<TPSolver>::QCIterator QCIterator;
00088   QCDistMDSystem (void);
00092   QCDistMDSystem(int& argc, char **&  argv);
00096   ~QCDistMDSystem (void);
00100   bool isMine (int sbdId) const { 
00101     return ( sbdId >= QCDistMDSystem<TPSolver>::offset &&
00102           sbdId <  QCDistMDSystem<TPSolver>::offset + QCDistMDSystem<TPSolver>::nbDomains ); 
00103   }
00104 
00105 
00106 
00110   int getNproc (void) const  { return nproc; }
00114   int getRank (void) const { return myrank; }
00118   int getProcRank (int sd) const {
00119     int q = QCMDSystem<TPSolver>::totalNbDomains / nproc;
00120     int r = QCMDSystem<TPSolver>::totalNbDomains % nproc;
00121     if (sd < (q+1)*r) {
00122       return sd / (q+1);
00123     } 
00124     return (sd-r) / q;
00125   }
00129   void readPartition (const string& name, const string& path);
00133   void allocDomains (int nb);
00137   void allocStructures (int sbdId, int nbatoms, int nbAO, int nbovlp, int *sbdIds, int *nbshared);
00141   void addAtom (const QCPoint3D& coords, int type, int globalIdx, int  domainIdx,
00142                 QCSubDomainZone  zone, int atomIdx);
00146   void fillMaps (int nbsbd, const int *domainIdxs, const QCSubDomainZone *zones,  const int *atomIdxs);
00150   void fillMaps (const std::vector< QCAtomIn > & v );
00154   template <class TPManager>
00155   void init (TPManager&    manager, const string& path);
00156 #ifdef QC_REPORT_TIME
00157 
00160   inline void computeAverages (QCFloat& tResTime,
00161                                QCFloat& tFermiTime,
00162                                QCFloat& tDensLTime,
00163                                QCFloat& tDensCTime,
00164                                QCFloat& tFockLTime,
00165                                QCFloat& tFockCTime);
00166 #endif
00167 
00168 
00169 //   template <class TPManager>
00170 //   inline void buildMatrices (TPManager& manager);
00174   template <class TPManager>
00175   void completeHamiltonMatrices (TPManager& manager);
00179   template <class TPManager>
00180   QCFloat completeFockMatricesAndElecEnergy (TPManager& manager, bool isFirstCall, int iter);
00184   void getGlobalMinMax (QCFloat& min, QCFloat& max);
00188   void getTotalSumNbElecs (QCFloat * nbelec, int size);
00192   QCFloat getTotalSumTrace (void);
00196   void removeLevels (QCFloat& sumNbElecs, QCTopEnergyLevel *topEnergyLevels);
00200   void initFermiSeq (void);
00204   template <class TPMDSystem>
00205   void adjustFermiEnergySeq (TPMDSystem& mdsystem, QCMemory& memory);
00206   
00210   void addLevels (QCFloat& sumNbElecs, const int * nbOM,
00211                   QCTopEnergyLevel * topEnergyLevels);
00212 
00213   /*********************************************************/
00214   /*********************************************************/
00218   template <class TPManager>
00219   void completeDensityMatrices (TPManager& manager);
00220   /*********************************************************/
00221   /*********************************************************/
00225   template <class TPParam>
00226   QCFloat applyOptimalDamping (const TPParam * QCRestrict params);
00230 
00231 
00232 
00233 
00234 
00235 
00236 
00237   bool testConvergence (QCFloat& error, QCFloat& threshold);
00241   QCFloat computeSumOfEigenValOccupied ();
00252   template <class TPManager>
00253   void writeDensityOnFileAscii(TPManager& manager,  const string& path);
00259   void readDensityFromFileAscii(const QCFiles & files,  const string& path);
00260 
00261   //
00263 protected:
00264 
00265   struct QCComm;
00269   void  allocBuffers (bool density);
00273   void  freeBuffers (void);
00277   void  fillCommunicationStruct(const int &);
00281   void  fillSDsCommArray (void);
00285   void  fillSDsCommVect (void);
00286   
00288 private:
00292   void initMPI(int& argc, char **& argv);
00296   void postDensitySends (int currSend, int * sendInfo, MPI_Request *requests);
00300   void postDensityRecvs (int currRecv, int recvIdx, int * recvInfo, MPI_Request * requests);
00304   template <class TPParam>
00305   void mergeDensityAux (const TPParam * QCRestrict params,
00306                         QCSubDomain&    QCRestrict contributer,
00307                         QCSymMatrix&    QCRestrict remWeightedDensity,
00308                         QCMatrix&       QCRestrict wDensityAB,
00309                         QCMatrix&       QCRestrict interPAB,
00310                         QCSymMatrix&    QCRestrict wDensityAA);
00311   template <class TPParam>
00312   void mergeDensityAux (const TPParam * QCRestrict params,
00313                         QCSubDomain&    QCRestrict contributer1,
00314                         QCSubDomain&    QCRestrict contributer2,
00315                         QCSymMatrix&    QCRestrict remWeightedDensity,
00316                         QCMatrix&       QCRestrict wDensityAB,
00317                         QCMatrix&       QCRestrict interPAB,
00318                         QCSymMatrix&    QCRestrict wDensityAA);
00322   void printProcArray (int * array, bool issend) const;
00326   void printProcVect (const vector<QCComm>& vect,  bool issend) const;
00329 public:
00330 protected:
00334   class QCComm {
00335     friend ostream &operator<<(ostream & out, const QCComm & Comm)
00336     { out << " ("<<Comm.proc<< ","<<Comm.sbdId << ","<<Comm.step <<") " ; return out ;}
00337   public:
00338     QCComm () : proc(-1), sbdId(-1), step(-1), time(QC_ZERO)
00339     {}    
00340     QCComm (int _proc , int _sbdId , int _step ) : proc(_proc),
00341                                                    sbdId(_sbdId), step(_step), time(QC_ZERO)
00342     {}
00343     QCComm (const QCComm &Com) : proc(Com.proc), sbdId(Com.sbdId), step(Com.step), time(Com.time)
00344     {}
00345     ~QCComm () 
00346     {}
00347     const QCComm & operator=(const QCComm &Com)
00348     { this->proc = Com.proc ; this->sbdId = Com.sbdId; this->step = Com.step; this->time = Com.time ; return *this;}
00349     //
00350     int     proc;
00351     int     sbdId;
00352     int     step;
00353     QCFloat time;
00354   };
00355 
00356 
00357 private:
00358 
00362   int myrank;
00366   int cartrank;
00370   int nproc;
00374   int maxOvrlpSize;
00378   int maxOvrlpSDs;
00382   int maxNbAtom;
00386   int maxNbAO;
00390   int maxDensitySize;
00394     // pour manipuler les buffers
00395   QCSubDomain * pipeline;
00396 
00400   QCFloat             ** atomsBuffers;
00401   
00402   int                 ** orderBuffers;
00403   
00404   int                 ** ovrlpBuffers;
00405   
00406   QCSubDomainZone     ** zoneBuffers;
00407 
00408   int                 ** blockBuffers;
00409 
00410   QCFloat             ** densBuffers;
00411 
00412   QCFloat             *  wdensMatBuffs[2];
00416   MPI_Comm            QC_COMM_WORLD;
00417 
00418   MPI_Op              QC_MIN_MAX;
00419 
00420   MPI_Datatype        QC_2DOUBLE ;
00421   
00425   int                 * sdsFromProc;
00426   int                 * sdsIntoProc;
00427   //
00428 //   vector<QCComm>      sends;
00429 //   vector<QCComm>      recvs;
00430   //
00433   int                 _nbLocalContrib ;
00436   int *                _localContrib ;
00439   std::vector<QCComm>  _sends;
00440   std::vector<QCComm>  _recvs;
00441   std::map<int,int>    _globalToLocalNum ;
00442   int                  cptIndex ;
00443   //
00444   int                 remSendsIdx;
00445   int                 remRecvsIdx;
00446 
00447 
00451 
00452 
00453   QCFloatFloat * rbuffer;
00454   QCFloatFloat * sbuffer;
00455   int          * rcounts;
00456   int          * displs;
00457   int          * nbOM;
00458   MPI_Datatype   stype;
00459   //
00461   int            totalAO;
00464   int            _totalAOAll;
00465 
00466 };
00467 
00468 
00469 
00470 
00471 #ifdef QC_REPORT_TIME
00472 
00475 template <class TPSolver>
00476 inline void 
00477 QCDistMDSystem<TPSolver>::computeAverages (QCFloat& tResTime,
00478                                            QCFloat& tFermiTime,
00479                                            QCFloat& tDensLTime,
00480                                            QCFloat& tDensCTime,
00481                                            QCFloat& tFockLTime,
00482                                            QCFloat& tFockCTime)
00483 {
00484   
00485   QCFloat in[6], out[6];
00486   
00487   in [0]  = tResTime;
00488   in [1]  = tFermiTime;
00489   in [2]  = tDensLTime;
00490   in [3]  = tDensCTime;
00491   in [4]  = tFockLTime;
00492   in [5]  = tFockCTime;
00493 
00494   MPI_Reduce(in, out, 6, MPI_DOUBLE, MPI_MAX, 0, QC_COMM_WORLD);
00495 
00496   tResTime   = out [0] / QCMDSystem<TPSolver>::totalNbDomains;
00497   tFermiTime = out [1];
00498   tDensLTime = out [2] / QCMDSystem<TPSolver>::totalNbDomains;
00499   tDensCTime = out [3];
00500   tFockLTime = out [4] / QCMDSystem<TPSolver>::totalNbDomains;
00501   tFockCTime = out [5];
00502 
00503 }
00504 
00505 
00506 #endif
00507 
00508 
00509 
00510 
00511 // template <class TPSolver>
00512 // template <class TPManager>
00513 // inline void
00514 // QCDistMDSystem<TPSolver>::buildMatrices (TPManager & manager) {
00515 //   //
00516 //   QCIterator workingSystem;
00517 //   for (workingSystem  = QCMDSystem<TPSolver>::begin(); 
00518 //        workingSystem != QCMDSystem<TPSolver>::end();  ++workingSystem) {
00519 //      workingSystem->QCModelMatrices::buildMatrices(manager, workingSystem);
00520 //    }
00521 //  }
00522 
00523 
00524 #endif // HAVE_MPI
00525 
00526 
00527 
00528 
00529 #endif //QC_DIST_MD_SYSTEM_H_

Generated on Sat Jan 28 21:07:26 2006 for QC++ by  doxygen 1.4.4