00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020 #ifndef QC_DIST_MD_SYSTEM_H_
00021 #define QC_DIST_MD_SYSTEM_H_
00022
00023
00024 #include <vector>
00025 #include <map>
00026
00027 #include "QCCommon.hpp"
00028 #include "QCMDSystem.hpp"
00029
00030
00031 #if defined(HAVE_MPI) && defined(WITH_MPI_SUPPORT)
00032
00033 # include <mpi.h>
00034
00035 # define HCONTRIB_ATOM 0
00036 # define HCONTRIB_ORDER 1
00037 # define HCONTRIB_OVRLP 2
00038 # define HCONTRIB_ZONE 3
00039 # define HCONTRIB_DENS 4
00040
00041
00042 # define FCONTRIB_ATOM (0 + HCONTRIB_DENS)
00043 # define FCONTRIB_ORDER (1 + HCONTRIB_DENS)
00044 # define FCONTRIB_OVRLP (2 + HCONTRIB_DENS)
00045 # define FCONTRIB_ZONE (3 + HCONTRIB_DENS)
00046 # define FCONTRIB_BLOCK (4 + HCONTRIB_DENS)
00047 # define FCONTRIB_DENS (5 + HCONTRIB_DENS)
00048
00049
00050 # define DCONTRIB_INFO (0 + FCONTRIB_DENS)
00051 # define DCONTRIB_ORDER (1 + FCONTRIB_DENS)
00052 # define DCONTRIB_OVRLP (2 + FCONTRIB_DENS)
00053 # define DCONTRIB_DENS (3 + FCONTRIB_DENS)
00054
00055
00056 # define SDS_HEAD 1
00057 # define NB_SDS 0
00058
00059
00060
00064 struct QCFloatInt {
00065
00066 QCFloat floatval;
00067
00068 int intval;
00069 };
00070
00075 template <class TPSolver>
00076 class QCDistMDSystem : public QCMDSystem<TPSolver>
00077 {
00078
00080 public:
00084 typedef typename QCMDSystem<TPSolver>::QCIterator QCIterator;
00088 QCDistMDSystem (void);
00092 QCDistMDSystem(int& argc, char **& argv);
00096 ~QCDistMDSystem (void);
00100 bool isMine (int sbdId) const {
00101 return ( sbdId >= QCDistMDSystem<TPSolver>::offset &&
00102 sbdId < QCDistMDSystem<TPSolver>::offset + QCDistMDSystem<TPSolver>::nbDomains );
00103 }
00104
00105
00106
00110 int getNproc (void) const { return nproc; }
00114 int getRank (void) const { return myrank; }
00118 int getProcRank (int sd) const {
00119 int q = QCMDSystem<TPSolver>::totalNbDomains / nproc;
00120 int r = QCMDSystem<TPSolver>::totalNbDomains % nproc;
00121 if (sd < (q+1)*r) {
00122 return sd / (q+1);
00123 }
00124 return (sd-r) / q;
00125 }
00129 void readPartition (const string& name, const string& path);
00133 void allocDomains (int nb);
00137 void allocStructures (int sbdId, int nbatoms, int nbAO, int nbovlp, int *sbdIds, int *nbshared);
00141 void addAtom (const QCPoint3D& coords, int type, int globalIdx, int domainIdx,
00142 QCSubDomainZone zone, int atomIdx);
00146 void fillMaps (int nbsbd, const int *domainIdxs, const QCSubDomainZone *zones, const int *atomIdxs);
00150 void fillMaps (const std::vector< QCAtomIn > & v );
00154 template <class TPManager>
00155 void init (TPManager& manager, const string& path);
00156 #ifdef QC_REPORT_TIME
00157
00160 inline void computeAverages (QCFloat& tResTime,
00161 QCFloat& tFermiTime,
00162 QCFloat& tDensLTime,
00163 QCFloat& tDensCTime,
00164 QCFloat& tFockLTime,
00165 QCFloat& tFockCTime);
00166 #endif
00167
00168
00169
00170
00174 template <class TPManager>
00175 void completeHamiltonMatrices (TPManager& manager);
00179 template <class TPManager>
00180 QCFloat completeFockMatricesAndElecEnergy (TPManager& manager, bool isFirstCall, int iter);
00184 void getGlobalMinMax (QCFloat& min, QCFloat& max);
00188 void getTotalSumNbElecs (QCFloat * nbelec, int size);
00192 QCFloat getTotalSumTrace (void);
00196 void removeLevels (QCFloat& sumNbElecs, QCTopEnergyLevel *topEnergyLevels);
00200 void initFermiSeq (void);
00204 template <class TPMDSystem>
00205 void adjustFermiEnergySeq (TPMDSystem& mdsystem, QCMemory& memory);
00206
00210 void addLevels (QCFloat& sumNbElecs, const int * nbOM,
00211 QCTopEnergyLevel * topEnergyLevels);
00212
00213
00214
00218 template <class TPManager>
00219 void completeDensityMatrices (TPManager& manager);
00220
00221
00225 template <class TPParam>
00226 QCFloat applyOptimalDamping (const TPParam * QCRestrict params);
00230
00231
00232
00233
00234
00235
00236
00237 bool testConvergence (QCFloat& error, QCFloat& threshold);
00241 QCFloat computeSumOfEigenValOccupied ();
00252 template <class TPManager>
00253 void writeDensityOnFileAscii(TPManager& manager, const string& path);
00259 void readDensityFromFileAscii(const QCFiles & files, const string& path);
00260
00261
00263 protected:
00264
00265 struct QCComm;
00269 void allocBuffers (bool density);
00273 void freeBuffers (void);
00277 void fillCommunicationStruct(const int &);
00281 void fillSDsCommArray (void);
00285 void fillSDsCommVect (void);
00286
00288 private:
00292 void initMPI(int& argc, char **& argv);
00296 void postDensitySends (int currSend, int * sendInfo, MPI_Request *requests);
00300 void postDensityRecvs (int currRecv, int recvIdx, int * recvInfo, MPI_Request * requests);
00304 template <class TPParam>
00305 void mergeDensityAux (const TPParam * QCRestrict params,
00306 QCSubDomain& QCRestrict contributer,
00307 QCSymMatrix& QCRestrict remWeightedDensity,
00308 QCMatrix& QCRestrict wDensityAB,
00309 QCMatrix& QCRestrict interPAB,
00310 QCSymMatrix& QCRestrict wDensityAA);
00311 template <class TPParam>
00312 void mergeDensityAux (const TPParam * QCRestrict params,
00313 QCSubDomain& QCRestrict contributer1,
00314 QCSubDomain& QCRestrict contributer2,
00315 QCSymMatrix& QCRestrict remWeightedDensity,
00316 QCMatrix& QCRestrict wDensityAB,
00317 QCMatrix& QCRestrict interPAB,
00318 QCSymMatrix& QCRestrict wDensityAA);
00322 void printProcArray (int * array, bool issend) const;
00326 void printProcVect (const vector<QCComm>& vect, bool issend) const;
00329 public:
00330 protected:
00334 class QCComm {
00335 friend ostream &operator<<(ostream & out, const QCComm & Comm)
00336 { out << " ("<<Comm.proc<< ","<<Comm.sbdId << ","<<Comm.step <<") " ; return out ;}
00337 public:
00338 QCComm () : proc(-1), sbdId(-1), step(-1), time(QC_ZERO)
00339 {}
00340 QCComm (int _proc , int _sbdId , int _step ) : proc(_proc),
00341 sbdId(_sbdId), step(_step), time(QC_ZERO)
00342 {}
00343 QCComm (const QCComm &Com) : proc(Com.proc), sbdId(Com.sbdId), step(Com.step), time(Com.time)
00344 {}
00345 ~QCComm ()
00346 {}
00347 const QCComm & operator=(const QCComm &Com)
00348 { this->proc = Com.proc ; this->sbdId = Com.sbdId; this->step = Com.step; this->time = Com.time ; return *this;}
00349
00350 int proc;
00351 int sbdId;
00352 int step;
00353 QCFloat time;
00354 };
00355
00356
00357 private:
00358
00362 int myrank;
00366 int cartrank;
00370 int nproc;
00374 int maxOvrlpSize;
00378 int maxOvrlpSDs;
00382 int maxNbAtom;
00386 int maxNbAO;
00390 int maxDensitySize;
00394
00395 QCSubDomain * pipeline;
00396
00400 QCFloat ** atomsBuffers;
00401
00402 int ** orderBuffers;
00403
00404 int ** ovrlpBuffers;
00405
00406 QCSubDomainZone ** zoneBuffers;
00407
00408 int ** blockBuffers;
00409
00410 QCFloat ** densBuffers;
00411
00412 QCFloat * wdensMatBuffs[2];
00416 MPI_Comm QC_COMM_WORLD;
00417
00418 MPI_Op QC_MIN_MAX;
00419
00420 MPI_Datatype QC_2DOUBLE ;
00421
00425 int * sdsFromProc;
00426 int * sdsIntoProc;
00427
00428
00429
00430
00433 int _nbLocalContrib ;
00436 int * _localContrib ;
00439 std::vector<QCComm> _sends;
00440 std::vector<QCComm> _recvs;
00441 std::map<int,int> _globalToLocalNum ;
00442 int cptIndex ;
00443
00444 int remSendsIdx;
00445 int remRecvsIdx;
00446
00447
00451
00452
00453 QCFloatFloat * rbuffer;
00454 QCFloatFloat * sbuffer;
00455 int * rcounts;
00456 int * displs;
00457 int * nbOM;
00458 MPI_Datatype stype;
00459
00461 int totalAO;
00464 int _totalAOAll;
00465
00466 };
00467
00468
00469
00470
00471 #ifdef QC_REPORT_TIME
00472
00475 template <class TPSolver>
00476 inline void
00477 QCDistMDSystem<TPSolver>::computeAverages (QCFloat& tResTime,
00478 QCFloat& tFermiTime,
00479 QCFloat& tDensLTime,
00480 QCFloat& tDensCTime,
00481 QCFloat& tFockLTime,
00482 QCFloat& tFockCTime)
00483 {
00484
00485 QCFloat in[6], out[6];
00486
00487 in [0] = tResTime;
00488 in [1] = tFermiTime;
00489 in [2] = tDensLTime;
00490 in [3] = tDensCTime;
00491 in [4] = tFockLTime;
00492 in [5] = tFockCTime;
00493
00494 MPI_Reduce(in, out, 6, MPI_DOUBLE, MPI_MAX, 0, QC_COMM_WORLD);
00495
00496 tResTime = out [0] / QCMDSystem<TPSolver>::totalNbDomains;
00497 tFermiTime = out [1];
00498 tDensLTime = out [2] / QCMDSystem<TPSolver>::totalNbDomains;
00499 tDensCTime = out [3];
00500 tFockLTime = out [4] / QCMDSystem<TPSolver>::totalNbDomains;
00501 tFockCTime = out [5];
00502
00503 }
00504
00505
00506 #endif
00507
00508
00509
00510
00511
00512
00513
00514
00515
00516
00517
00518
00519
00520
00521
00522
00523
00524 #endif // HAVE_MPI
00525
00526
00527
00528
00529 #endif //QC_DIST_MD_SYSTEM_H_