/*
* Name: p4 multithreaded distributed verify
* Description: Divide's all remaining depots evenly based on the number of days it has left to run.
* The runs up to five verification checks at the same time.
* Programmers: Luke Howell and Kirk McCann
* Date: 12/12/2011
* P4API Version: 2010.2
* Version: 1.0
*/
#include "clientapi.h"
#include <iostream>
#include <fstream>
#include <sstream>
#include <locale>
#include <pthread.h>
#include <sys/stat.h>
#include <map>
#include <list>
#include <set>
// #define DEBUG
// #define DEBUG_FILE_MOD_DAY 5
// #define DEBUG_CURRENT_DAY 3
#define LOG_FILE "/var/log/p4verify_log"
#define ERROR_LOG "/var/log/p4verify_error_log"
#define PASSWD "<REMOVED>"
#define PORT "<REMOVED>"
#define USERNAME "<REMOVED>"
#define WEEKLY_START_DAY 0
#define WEEKLY_END_DAY 6
#define NUM_WEEKLY_RUNS 5
#define DAYS_TO_RUN 1, 2, 3, 4, 5
#define NUM_THREADS 5
using namespace std;
/*
* Name: Depot
* Description: Structure that will house all the information about a depot.
* Programmers: Luke Howell and Kirk McCann
*/
struct Depot {
string name;
string create_date;
string type;
string location;
string comment;
int size;
int files;
string verification;
bool operator<( const Depot& rhs ) const { return size < rhs.size; }
Depot()
{
size = 0;
files = 0;
verification = "";
}
};
pthread_t tCall[NUM_THREADS];
pthread_mutex_t mutex_ContainSizes;
pthread_mutex_t mutex_Verified;
pthread_barrier_t barr;
typedef map< string, Depot > DepotMap;
typedef list< Depot > DepotList;
typedef set< string > StringSet;
typedef list< string > StringList;
DepotMap NeedSizes; //Depot Location is the key
DepotList ContainSizes;
DepotMap ToBeVerified; //Name of the depot is the key
DepotMap Verified; //Name of the depot is the key
StringSet PreviouslyVerified; //Contains the name of the depots
StringList NeedSizeList_Location; //Contains the Location of the depots
StringList NeedVerifyList_Location; //Contains the Location of the depots
/*
* Class Name: MyClient User
* Description: Overloaded Client User Functions.
* Programmers: Luke Howell and Kirk McCann
*/
class MyClientUser : public ClientUser
{
/*
* Function Name: strpos
* Input: const char pointer containing the c string to be searched
char pointer containing the c string to search for
* Output: location where the c string starts or -1
* Description: Find the location of a c string in a c string
* Programmers: Luke Howell
*/
int strpos( const char *haystack, char *needle )
{
char *p = strstr( haystack, needle );
if( p )
return p - haystack;
return -1;
}
/*
* Function Name: ParseDepot
* Input: const char pointer containing results of an instance of the output of the p4 depots command
* Output: Null Return.
* Description: Creates an instance of the Depot struct containing name, create date, location, and comments.
* Adds the instance to the NeedSizes depot map using the location as the key.
* Adds the location to the NeedSizeList_Location list
* Programmers: Luke Howell
*/
void ParseDepot( const char *data )
{
char str[ 2048 ];
strcpy( str, data );
char *ch;
ch = strtok( str, " " );
int count = 1;
Depot depot;
while( ch != NULL )
{
switch( count++ )
{
case 1:
break;
case 2:
if( PreviouslyVerified.find( ch ) != PreviouslyVerified.end() )
return;
depot.name = ch;
break;
case 3:
depot.create_date = ch;
break;
case 4:
depot.type = ch;
break;
case 5:
depot.location = ch;
depot.location = "//" + depot.location;
break;
case 6:
depot.comment = ch;
break;
default:
depot.comment = depot.comment + " " + ch;
break;
}
ch = strtok( NULL, " " );
}
NeedSizes[ depot.location ] = depot;
NeedSizeList_Location.push_back( depot.location );
#ifdef DEBUG
cout << "DEBUG - Parsing depot '" << depot.name << "' completed." << endl;
#endif
return;
}
/*
* Function Name: ParseSize
* Input: const char pointer containing results of an instance of the output of the p4 sizes command
* Output: Null Return.
* Description: Tokenizes the output of the P4 Sizes command.
* Combines it with the data from the the NeedSizes Map.
* Adds the instance to the ContainSizes Depot List.
* Programmers: Luke Howell
*/
void ParseSize( const char *data )
{
char str[ 2048 ];
strcpy( str, data );
char *ch;
ch = strtok( str, " " );
int count = 1;
Depot depot;
while( ch != NULL )
{
switch( count++ )
{
case 1:
depot = NeedSizes[ ch ];
break;
case 2:
depot.files = atoi( ch );
break;
case 3:
break;
case 4:
depot.size = atoi( ch );
break;
case 5:
break;
case 6:
break;
default:
break;
}
ch = strtok( NULL, " " );
}
ContainSizes.push_back( depot );
#ifdef DEBUG
cout << "DEBUG - Parsing size for '" << depot.name << "' completed." << endl;
#endif
return;
}
/*
* Function Name: ParseVerify
* Input: const char pointer containing results of an instance of the output of the p4 verify command
* Output: Null Return.
* Description: Tokenizes the output of the P4 verify command.
* Saves it in the Verified Map.
* Programmers: Luke Howell
*/
void ParseVerify( const char *data )
{
char str[ 2048 ];
strcpy( str, data );
char *ch;
ch = strtok( str, "/" );
int count = 1;
Depot depot;
stringstream result;
while( ch != NULL )
{
switch( count++ )
{
case 1:
depot = Verified[ ch ];
result << depot.verification << data << endl;
depot.verification = result.str();
break;
default:
break;
}
ch = strtok( NULL, " " );
}
Verified[ depot.name ] = depot;
#ifdef DEBUG
cout << "DEBUG - Verifying '" << depot.name << "' completed." << endl;
#endif
return;
}
/*
* Function Name: OutputInfo
* Input: char containing the command level (Not Used)
* const char pointer containing results of an instance of the output of the p4 depot, size and verify commands
* Output: Null Return.
* Description: Tokenizes the output of the P4 command.
* Calls the appropriate function for what command was run.
* Programmers: Luke Howell
*/
void OutputInfo( char level, const char *data )
{
if( strpos( data, "Depot" ) == 0 )
ParseDepot( data );
else
{
char str[ 2048 ];
strcpy( str, data );
char *ch;
strtok( str, " " );
ch = strtok( NULL, " " );
if( !strcmp( ch, "-" ) )
ParseVerify( data );
else
ParseSize( data );
}
return;
}
/*
* Function Name: OutputError
* Input: const char pointer containing results of an instance of error output
* Output: Null Return.
* Description: Writes the error to a file
* Programmers: Luke Howell
*/
void OutputError( const char *data )
{
ofstream errorlog( ERROR_LOG, fstream::app );
errorlog << data << endl;
errorlog.close();
}
};
void clearNecessaryLog();
int getNumberOfRunsLeft();
void loadPreviouslyVerified();
void parseDepots();
void *getSizes(void*);
void loadDepotsToBeVerified( int DaysLeftToVerify );
void sortDepotsToBeVerified();
void *verifyDepots(void*);
int main( int argc, char **argv );
/*
* Function Name: clearNecessaryLog
* Input: N\A
* Output: N\A
* Description: Deletes the Log File containing the list of depots verified
* If it is the first day in the week the file should run or
* If the current day is less than the last day it was modified.
* Programmers: Luke Howell and Kirk McCann
*/
void clearNecessaryLog()
{
time_t rawtime;
struct tm *today;
struct tm *filemod;
struct stat attrib;
time( &rawtime );
today = localtime( &rawtime );
int currentDay = today->tm_wday;
#ifdef DEBUG_CURRENT_DAY
currentDay = DEBUG_CURRENT_DAY;
#endif
stat( LOG_FILE, &attrib );
filemod = gmtime( &( attrib.st_mtime ) );
int fileModDay = filemod->tm_wday;
#ifdef DEBUG_FILE_MOD_DAY
fileModDay = DEBUG_FILE_MOD_DAY;
#endif
int daysToRun[ NUM_WEEKLY_RUNS ] = { DAYS_TO_RUN };
#ifdef DEBUG
cout << "Current Day " << currentDay << endl;
cout << "Days to Run " << daysToRun[ 0 ] << endl;
cout << "FileModDay " << fileModDay << endl;
#endif
if( currentDay <= daysToRun[ 0 ] || currentDay < fileModDay )
{
cout << "Removing Log File" << endl;
remove( LOG_FILE );
}
}
/*
* Function Name: getNumberOfRunsLeft
* Input: N\A
* Output: returns the number of runs left (int) or
* 0 if it should not run today.
* Description: Gets the numerical day of the week
* Checks subtracts the number of days it has already run this week
* Checks to see if it should run today
* Returns either the number of days left to run or 0
* Programmers: Luke Howell
*/
int getNumberOfRunsLeft()
{
time_t rawtime;
struct tm *timeinfo;
time( &rawtime );
timeinfo = localtime( &rawtime );
int currentDay = timeinfo->tm_wday;
#ifdef DEBUG_CURRENT_DAY
currentDay = DEBUG_CURRENT_DAY;
#endif
int daysToRun[ NUM_WEEKLY_RUNS ] = { DAYS_TO_RUN };
int numberRunsLeft = NUM_WEEKLY_RUNS;
bool runToday = 0;
int whichRun = NUM_WEEKLY_RUNS - 1;
for( int i = 0; i < NUM_WEEKLY_RUNS; i++ )
{
if( daysToRun[ i ] < currentDay )
{
numberRunsLeft--;
}
else if( daysToRun[ i ] == currentDay )
{
runToday = true;
}
}
return runToday ? numberRunsLeft : 0;
}
/*
* Function Name: loadPreviouslyVerified
* Input: N\A
* Output: N\A
* Description: Reads the log file containing the list of verified depots (or creates it if it doesnt exist)
* Inserts the name of the depot in the PreviouslyVerified
* Programmers: Luke Howell
*/
void loadPreviouslyVerified()
{
ifstream logfile( LOG_FILE );
string name;
if( logfile.is_open() )
{
while( logfile.good() )
{
getline( logfile, name );
if( name.length() > 0 )
{
#ifdef DEBUG
cout << "DEBUG - " << name << " was previously verified." << endl;
#endif
PreviouslyVerified.insert( name );
}
}
}
else
{
//cout << "Error Opening File" << endl;
}
logfile.close();
return;
}
/*
* Function Name: parseDepots
* Input: N\A
* Output: N\A
* Description: Runs the P4 Depots command
* Programmers: Luke Howell and Kirk McCann
*/
void parseDepots()
{
MyClientUser ui;
ClientApi client;
StrBuf msg;
StrBuf passwd;
Error e;
passwd = PASSWD;
client.SetPort( PORT );
client.SetUser( USERNAME );
client.SetPassword( &passwd );
client.Init( &e );
if( e.Test() )
{
e.Fmt( &msg );
fprintf( stderr, "%s\n", msg.Text() );
return;
}
client.Run( "depots", &ui );
client.Final( &e );
if( e.Test() )
{
e.Fmt( &msg );
fprintf( stderr, "%s\n", msg.Text() );
return;
}
}
/*
* Function Name: getSizes
* Input: void pointer
* Output: void pointer
* Description: Distributes running the p4 sizes command (against the depots that have not been run) across multiple threads.
* Programmers: Kirk McCann
*/
void *getSizes(void *arg)
{
#ifdef DEBUG
cout << "DEBUG - GetSize " << pthread_self() << endl;
#endif
int ret = 0;
string location = "NULL";
int rc = pthread_barrier_wait(&barr);
if(rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD)
{
printf ("Could not wait on barrier\n");
exit(-1);
}
for(;;)
{
pthread_mutex_lock(&mutex_ContainSizes);
#ifdef DEBUG
cout << "DEBUG - GetSize " << pthread_self() << " processing:" << location << " count " << NeedSizeList_Location.size() << endl;
#endif
if (NeedSizeList_Location.size() == 0)
{
pthread_mutex_unlock( &mutex_ContainSizes );
#ifdef DEBUG
cout << "DEBUG - GetSize exit " << pthread_self() << endl;
#endif
pthread_exit(&ret);
}
location = NeedSizeList_Location.front();
NeedSizeList_Location.pop_front();
pthread_mutex_unlock( &mutex_ContainSizes);
#ifdef DEBUG
cout << "DEBUG - GetSize " << pthread_self() << " processing:" << location << " count " << NeedSizeList_Location.size() << endl;
#endif
MyClientUser ui;
ClientApi client;
StrBuf msg;
StrBuf passwd;
Error e;
string a;
passwd = PASSWD;
client.SetPort( PORT );
client.SetUser( USERNAME );
client.SetPassword( &passwd );
client.Init( &e );
if( e.Test() )
{
e.Fmt( &msg );
fprintf( stderr, "%s\n", msg.Text() );
ret = 1;
pthread_exit(&ret);
}
char *args[ 100 ];
args[0] = "-a";
args[1] = "-s";
char name[ location.length() + 1 ];
strcpy( name, location.c_str() );
args[2] = name;
client.SetArgv( 3, args );
client.Run( "sizes", &ui );
// Close connection
client.Final( &e );
if( e.Test() )
{
e.Fmt( &msg );
fprintf( stderr, "%s\n", msg.Text() );
ret = 1;
pthread_exit(&ret);
}
}
pthread_exit(&ret);
}
/*
* Function Name: loadDepotsToBeVerified
* Input: number of days left to verify (int)
* Output: void
* Description: Sorts the Depots Based on Size
* Loads the depots in order to the ToBeVerified map the key is the name;
* and stores the name in NeedVerifyList_Location;
* Programmers: Luke Howell
*/
void loadDepotsToBeVerified( int DaysLeftToVerify )
{
ContainSizes.sort( less< Depot >() );
int column = 1;
for( DepotList::iterator it = ContainSizes.begin(); it != ContainSizes.end(); it++ )
{
Depot depot = *it;
if( column == 1 )
{
ToBeVerified[ depot.name ] = depot;
NeedVerifyList_Location.push_back(depot.name);
}
column = column == DaysLeftToVerify ? 1 : column + 1;
}
#ifdef DEBUG
cout << "DEBUG - Loading depots to be verified completed." << endl;
#endif
return;
}
/*
* Function Name: verifyDepots
* Input: void pointer
* Output: void pointer
* Description: Distributes running the p4 verify command (against the depots that are to be runned on that day) across multiple threads.
* Programmers: Kirk McCann
*/
void *verifyDepots(void *arg)
{
#ifdef DEBUG
cout << " DEBUG - verifyDepots " << pthread_self() << endl;
#endif
int ret = 0;
string name = "NULL";
string location = "NULL";
int rc = pthread_barrier_wait(&barr);
if(rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD)
{
printf ("Could not wait on barrier\n");
exit(-1);
}
for(;;)
{
pthread_mutex_lock(&mutex_ContainSizes);
if (NeedVerifyList_Location.empty())
{
pthread_mutex_unlock( &mutex_ContainSizes );
#ifdef DEBUG
cout << "DEBUG - verifyDepots exit " << pthread_self() << endl;
#endif
pthread_exit(&ret);
}
name = NeedVerifyList_Location.front();
NeedVerifyList_Location.pop_front();
pthread_mutex_unlock( &mutex_ContainSizes);
#ifdef DEBUG
cout << "DEBUG - verifyDepots " << pthread_self() << " processing:" << location << " count " << NeedVerifyList_Location.size() << endl;
#endif
MyClientUser ui;
ClientApi client;
StrBuf msg;
StrBuf passwd;
Error e;
string a;
passwd = PASSWD;
client.SetPort( PORT );
client.SetUser( USERNAME );
client.SetPassword( &passwd );
client.Init( &e );
if( e.Test() )
{
e.Fmt( &msg );
fprintf( stderr, "%s\n", msg.Text() );
ret = 1;
pthread_exit(&ret);
}
Verified[ name ] = ToBeVerified[ name ];
location = Verified[ name ].location;
char *args[ 100 ];
args[0] = "-z";
args[1] = "-q";
char name[ location.length() + 1 ];
strcpy( name, location.c_str() );
args[2] = name;
client.SetArgv( 3, args );
client.Run( "verify", &ui );
#ifdef DEBUG
cout << "DEBUG - Verifying '" << location << "' completed." << endl;
#endif
// Close connection
client.Final( &e );
if( e.Test() )
{
e.Fmt( &msg );
fprintf( stderr, "%s\n", msg.Text() );
ret = 1;
pthread_exit(&ret);
}
}
pthread_exit(&ret);
}
/*
* Function Name: recordVerifiedDepots
* Input: void
* Output: void
* Description: Appends all depots verified to log file
* Programmers: Luke Howell
*/
void recordVerifiedDepots()
{
ofstream logfile( LOG_FILE, fstream::app );
for( DepotMap::iterator it = Verified.begin(); it != Verified.end(); it++ )
{
Depot depot = it->second;
logfile << depot.name << endl;
}
logfile.close();
}
/*
* Function Name: Main
* Input: int argc, char **argv (not used)
* Output: int (not used)
* Description: Call function to clear log
* Call function to get number of runs (if it shouldnt run error and exit)
* Call function to load previously verified Depots
* Call function to parse the depots
* Call function to get sizes for each thread
* join each thread for get sizes
* Call function to load the depots to be verified
* Call function to verify depots for each thread
* join each thread for verifing the depots
* Call function to log depots verified
* print each depot with details to screen
* Programmers: Luke Howell and Kirk McCann
*/
int main( int argc, char **argv )
{
int rc;
void *status;
pthread_mutex_init(&mutex_ContainSizes, NULL);
pthread_mutex_init(&mutex_Verified, NULL);
if (pthread_barrier_init(&barr, NULL, NUM_THREADS))
{
printf("Could not create a barrier\n");
return -1;
}
#ifdef DEBUG
cout << "DEBUG - Clearing necessary log files." << endl;
#endif
clearNecessaryLog();
#ifdef DEBUG
cout << "DEBUG - Clearing necessary log files." << endl;
#endif
#ifdef DEBUG
cout << "DEBUG - Determining number of runs left." << endl;
#endif
int numberRunsLeft = getNumberOfRunsLeft();
#ifdef DEBUG
cout << "DEBUG - Determining number of runs left completed." << endl;
#endif
if( !numberRunsLeft )
{
cout << "Should not run today." << endl;
return( 0 );
}
#ifdef DEBUG
cout << "DEBUG - Loading previously verified depots." << endl;
#endif
loadPreviouslyVerified();
#ifdef DEBUG
cout << "DEBUG - Loading previously verified depots completed." << endl;
#endif
#ifdef DEBUG
cout << "DEBUG - Parsing depots." << endl;
#endif
parseDepots();
#ifdef DEBUG
cout << "DEBUG - Parsing depots completed." << endl;
#endif
#ifdef DEBUG
cout << "DEBUG - Parsing sizes of depots." << endl;
#endif
for (int i=0;i<NUM_THREADS;i++)
{
rc = pthread_create(&tCall[i], NULL, getSizes, (void *) i);
}
//Join all the threads
for (int i=0;i<NUM_THREADS;i++)
{
pthread_join(tCall[i], &status);
}
#ifdef DEBUG
cout << "DEBUG - Parsing sizes of depots completed." << endl;
#endif
#ifdef DEBUG
cout << "DEBUG - Loading depots to be verified." << endl;
#endif
loadDepotsToBeVerified( numberRunsLeft );
#ifdef DEBUG
cout << "DEBUG - Loading depots to be verified completed." << endl;
#endif
#ifdef DEBUG
cout << "DEBUG - Verifying depots." << endl;
#endif
for (int i=0;i<NUM_THREADS;i++)
{
rc = pthread_create(&tCall[i], NULL, verifyDepots, (void *) i);
}
//Join all the threads
for (int i=0;i<NUM_THREADS;i++)
{
pthread_join(tCall[i], &status);
}
#ifdef DEBUG
cout << "DEBUG - Verifying depots completed." << endl;
#endif
// Close connection
recordVerifiedDepots();
for( DepotMap::iterator it = Verified.begin(); it != Verified.end(); it++ )
{
Depot depot = it->second;
cout << "----------------------------------" << endl;
cout << "Name: " << depot.name << endl;
cout << "Date Created: " << depot.create_date << endl;
cout << "Type: " << depot.type << endl;
cout << "Location: " << depot.location << endl;
cout << "Number of Files: " << depot.files << endl;
cout << "Size of Depot (bytes): " << depot.size << endl;
cout << "Verification Info" << endl;
cout << "----------------------------------" << endl;
cout << depot.verification << endl;
cout << "----------------------------------" << endl;
}
return 0;
}