#include <iostream>
#include <exception>
#include <string>
#include <cstring> 
using namespace std;
#include <stdlib.h>

#include <mpi.h>

  #include <unistd.h>
//
//
// rotate exercies from nci
// passes a number around the ranks of a MPI program
// used to test connectivity. We use non block primitives just because we can
//

void do_work_non_blocking();

int mpirank=0;

const int MES_TAG=1;


#include<iostream>
#include<fstream>
#include<string>
#include<cstdlib>
#include<sstream>

std::string ssystem (const char *command) {
    char tmpname [L_tmpnam];
    std::tmpnam ( tmpname );
    std::string scommand = command;
    std::string cmd = scommand + " >> " + tmpname;
    std::system(cmd.c_str());
    std::ifstream file(tmpname, std::ios::in );
    std::string result;
        if (file) {
      while (!file.eof()) result.push_back(file.get());
          file.close();
    }
    remove(tmpname);
    return result;
}


int main( int argc, char* argv[])
	{
	try 
		{
		cout <<"rotate program called on host ";
       	size_t BUFFLEN= 256;
		char hostBuffer[BUFFLEN];
		hostBuffer[0]='\0';
		gethostname(hostBuffer, BUFFLEN);
		cout<<hostBuffer<<endl;


		int mpierror, mpisize;



        cout<<"Before MPI_Init"<<endl;
		mpierror = MPI_Init(&argc,&argv);
		if (mpierror != MPI_SUCCESS)
			{
			cerr <<"Error in mpi init "<<mpierror<<endl;
			exit(mpierror);
			}
        cout<<"Before MPI_Comm_size"<<endl;
		mpierror = MPI_Comm_size(MPI_COMM_WORLD,&mpisize);
		if (mpierror != MPI_SUCCESS)
			{
			cerr <<"Error in mpi comm size "<<mpierror<<endl;
			exit(mpierror);
			}
	    cout<< "Mpi size is "<<mpisize<<endl;
		mpierror = MPI_Comm_rank(MPI_COMM_WORLD,&mpirank);
		if (mpierror != MPI_SUCCESS)
			{
			cerr <<"Error in mpi rank size "<<mpierror<<endl;
			exit(mpierror);
			}
	        //std::string CPU_Affinity = ssystem("cat /proc/self/status | grep -i cpus_allowed_list");		
	    //std::string CPU_Affinity = ssystem("grep -i cpus_allowed_list /proc/self/status");		
		//cout<<"Hostname="<<hostBuffer<< ": Mpi rank  is "<<mpirank<<" and mpusize is "<<mpisize<< " and CPU Affinity is "<<CPU_Affinity<<endl;
		cout<<"Hostname="<<hostBuffer<< ": Mpi rank  is "<<mpirank<<" and mpusize is "<<mpisize<< endl;

		//
		//a do work here
		// 	
		do_work_non_blocking();

		cout <<"Before MPI_Finalize  from rank "<<mpirank<<endl;
		MPI_Finalize();
		cout <<"Exit from rank "<<mpirank<<endl;
		}
	catch (exception& e)
  		{
    		cerr<<"Exception caught " << e.what()<< "from mpirank "<<mpirank << endl;
  		}
	}//main

//------------------------------------------
void sendString(char* s, int destRank)
	{
	if (s==NULL)
		{
		cerr<<"MPI_Send. error null pointer sent!\n";
		return;
		}
	cout<<"MPI_Send::["<<mpirank<<"=>"<<destRank<< "] string is \""<<s<<"\""<<endl;
	MPI_Status status;
	int error=MPI_Ssend(s,strlen(s)+1,MPI_CHAR,destRank,MES_TAG,MPI_COMM_WORLD);
	if (error != MPI_SUCCESS)
		{
		cout<<"error  MPI_Ssend: from "<<mpirank<<" to "<<destRank<<endl;
		return; 
		}
	cout<<"........MPI_Send::["<<mpirank<<"=>"<<destRank<< "] Successful send\n";
	}//sendString
//------------------------------------------
void recvString(char* buffer, int MAX_BUFFER, int destRank)
	{
	cout<<"........MPI_Recv: rank "<<mpirank<< "<="<<destRank<<endl;
	MPI_Status status;
	int error=MPI_Recv(buffer,MAX_BUFFER,MPI_CHAR,destRank, MPI_ANY_TAG,MPI_COMM_WORLD,&status);
                       
	if (error != MPI_SUCCESS)
		{
		cout<<"error  MPI_Recv: from "<<mpirank<<endl;
		return; 
		}
	int received;
	MPI_Get_count(&status,MPI_CHAR,&received);
	if (strlen(buffer)==0)
		{
		cerr<<"Error from rank "<<mpirank<<" no string found!  "<<endl; 
		return;
		}
	if (strlen(buffer)<MAX_BUFFER)
		{
		cout <<"MPI_Recv["<<mpirank<< "<=" <<status.MPI_SOURCE << "]: Successful receive of \""<<buffer<<"\" bytes\n"; 
		}
	}//recvString

void do_work_non_blocking()
{
    cout<< "do_work_non_blocking: "<<mpirank<<endl;
	int error;
	double start = MPI_Wtime();
	bool finish=false;
	int MPI_SIZE;
	MPI_Comm_size(MPI_COMM_WORLD,&MPI_SIZE);
	
    char  my_cpu_name[BUFSIZ];
    int my_name_length;
    MPI_Request requestSend;
    MPI_Request requestReceive;
    
    MPI_Status status;
    int sendBuffer;
    int receiveBuffer;
    
    int nextRank;
    if (mpirank==MPI_SIZE-1)
    {
        nextRank=0;
    }
    else
    {
        nextRank=mpirank+1;
    }
    
    MPI_Get_processor_name(my_cpu_name, &my_name_length);
    cout<<"Rank "<<mpirank<<" Processor name is "<<my_cpu_name<<endl;
    
    bool finished=false;
    int currentRank=mpirank;
    while (!finished)
        {  
        //send rank
        sendBuffer=currentRank;
	    cout<<"Send from rank  "<<mpirank<<"=>"<<nextRank<<"  sendBuffer "<<sendBuffer<<endl;
        error =MPI_Isend(&sendBuffer,1,MPI_INT,nextRank,  MES_TAG,MPI_COMM_WORLD,&requestSend);
            
        if (error != MPI_SUCCESS)
            {
            cout<<"error  MPI_Isend(int): from "<<mpirank<<endl;
            return;
            }
            
         //receive rank
         error=MPI_Irecv(&receiveBuffer,1,MPI_INT,MPI_ANY_SOURCE,MPI_ANY_TAG,MPI_COMM_WORLD,&requestReceive);
         if (error != MPI_SUCCESS)
            {
            cout<<"error  MPI_Irecv(int): from "<<mpirank<<endl;
            return;
            }
         //wait
         MPI_Status status;
         cout<<"Before MPI_Wait for rank "<<mpirank<<endl;
         MPI_Wait(&requestSend,&status);
         MPI_Wait(&requestReceive,&status);
         cout<<"2*waits finished for rank "<<mpirank<<" and received value is "<<receiveBuffer<<endl;
         currentRank=receiveBuffer;
         if (currentRank==mpirank)
		    {
		    finished=true;
		    cout<<"SUCCESS from "<<mpirank<<endl;
		    }
            
        }//while
    double end = MPI_Wtime();
	cout<<"Time of work["<<	mpirank << "] "<< (end-start) << " seconds "<<endl;
}

//------------------------------------------

//------------------------------------------