#!/usr/local/bin/perl
# -*-Fundamental-*-

#
#   Copyright (c) 1996 Network Appliance, Inc.
#
#   You may distribute under the terms of the Artistic License, as
#   specified in the README file included in the ttt distribution.
#

# $Id: //depot/tools/main/p4bench/p4watch#2 $

#  Notes:
#
#  "p4watch" is intended to be used in tracking down Perforce
#  performance problems, whem it is suspected that something is
#  periodically causing Perforce commands which _should_ be quick to
#  execute to get "the slows".
#
#  It works by periodically running a (nominally) short-running "p4"
#  command, ("p4 describe 1", by default), and watching for the
#  command to complete or yeild output.  If the command has not
#  completed, or produced any output within a given timeout duration,
#  then a "p4wd" is done, in order to log system state.
#  
#  When the "p4wd" has been run, it will also then be run after the
#  original "p4" probe command finally does complete.
#
#  Optionally, a vmstat on the server host and a sysstat on the filer
#  (each of which produce periodic output) can be requested.
#
#  All output from all of this goes to the standard output, and
#  the ttt logfile.
# 
#  p4watch" requires the "ttt" library (which comes with it) and the
#  Time::HiRes perl module (available from CPAN). Also, you'll need a
#  working "p4wd" on your p4d host.
#
#  If you choose to enable the vmstat logging, the "vmstat" command
#  issued may need to be adjusted for the operating system you are running
#  your Perforce server on.
#
#  "p4watch" can be run from any host or as any user, provided that:
#
#     - you'll need to run on a host (and as a user) from which you can
#       run "rsh <p4dhost> p4wd".
#
#     - if you enable the "sysstatint" (filer sysstat output), you'll
#       need to run "p4watch" form a host (and as a user) that
#       can run "rsh <filer> sysstat".
#
#     - if you enable the "vmstatint" (p4d host sysstat logging),
#       you'll need to run "p4watch" from a host (and as a user) that
#       can run "rsh <p4dhost> vmstat".
#       
#  All "p4watch" arguments can be set either on the command line,
#  using, e.g.,
#
#    p4watch p4wdthresh=10 srv_host=charley user=rmg p4cmdint=120
#
#  To get a brief description of the argument (and their defaults),
#  use
#
#    p4watch =options
#
#  Here's more about the options:
#
#    p4cmd              the short running p4 command to run as a probe
#
#    p4wdthresh         threshold above which to run p4wd
#       i.e., if the "$p4dcmd" doesn't finish or yeild output for
#       this many seconds, run a p4wd.
#
#    vmstatint          vmstat interval argument
#       setting this causes "vmstat <vmstatint>" to be run on the p4d host
#
#    sysstatint         filer sysstat interval argument
#       setting this causes "sysstat <sysstatint> to be run on the filer
#
#    p4cmdint           p4 describe interval
#       this is the frequency at which "p4 describe" probes should be run
#
#    srv_host           host p4d is running on
#       The host where "p4d" is running
#
#    srv_port           port p4d is listening on
#       The port on srv_host where p4d is listening
#
#    srv_user           user to run as on srv_host
#       all command on srv_host will be run with "rsh -l <srv_user> ..."
#
#    cli_host           host to run "p4 describe" probes on
#       The host where the "p4 describes" should be run
#
#    cli_user           user to run as on cli_host
#       all command on cli_host will be run with "rsh -l <cli_user> ..."
#
#    filer              the filer hostname
#       
#    filer_user           user to run as on the filer
#       all command on the filer will be run with "rsh -l <filer_user> ..."
#
#  I suggest that, before attempting to run "p4watch", try running
#  "testAgents"; the output should look like
#
#  rmg $ testAgents
#          : logging to: /home/rmg/web/richard_geiger/guest/richard_geiger/p4bench/ttt_log
#  execute : 14498 on peggyo: date
#  execute : 14499 on peggyo: date
#  execute : 14500 on peggyo: date
#  14500   : STDOUT: Thu Sep 13 17:58:01 PDT 2001
#  14498   : STDOUT: Thu Sep 13 17:58:01 PDT 2001
#  14499   : STDOUT: Thu Sep 13 17:58:01 PDT 2001
#  14500   : STDERR: EOF
#  14500   : STDOUT: EOF
#  14498   : STDERR: EOF
#  14498   : STDOUT: EOF
#  14499   : STDERR: EOF
#  14499   : STDOUT: EOF
#  14498   : exit status = 0
#  14499   : exit status = 0
#  14500   : exit status = 0
#  exit    : pass
#
# And then, perhaps another one with one of the "date" commands to be run on the
# p4dhost and ("computer", in this example) and filer ("filer", in this example):
#
#    TBD - example
#
# (This will verify that the "ttt" library is working right; I think it should work fine
# on most Unixes)
#
# When "testAgents" seems to be running, you can try "p4watch"....
#

use Time::HiRes qw( usleep ualarm gettimeofday tv_interval );

my $t0;

sub mark_time
{
  my($msg, $init) = @_;

  if ($init eq "start")
    {
      &ttt_msg("mark_st", "$msg\n");
      $t0 = [gettimeofday];
      return;
    }
  
  $e = tv_interval ($t0);
  my @t1 = times;

  my $times = sprintf "%8.2f seconds", $e;

  &ttt_msg("mark_tm", "$times : $msg\n");

  $t0 = [gettimeofday];
  @t0 = @t1;
  return $e;
}

      
sub opts
{
  &ttt_add_opt("p4wdthresh", "^\\\\d+\\\$", 5, "threshold above which to run p4wd");
  &ttt_add_opt("vmstatint", "^\\\\d+\\\$", 10, "vmstat interval argument");
  &ttt_add_opt("sysstatint", "^\\\\d+\\\$", 10, "filer sysstat interval argument");
  &ttt_add_opt("p4cmd", ".*", "p4 describe 1", "the short running p4 command to run as a probe");
  &ttt_add_opt("p4cmdint", "^\\\\d+\\\$", 120, "p4 describe interval");
  &ttt_add_opt("srv_host", "^", "localhost", "host p4d is running on");
  &ttt_add_opt("srv_port", "^\\\\d+\\\$", "1666", "port p4d is listening on");
  &ttt_add_opt("srv_user", "^", "$user", "user to run as on srv_host");
  &ttt_add_opt("cli_host", "^", "localhost", "host to run p4cmd probes on");
  &ttt_add_opt("cli_user", "^", "$user", "user to run as on cli_host");
  &ttt_add_opt("filer", "^", "filer", "the hostname of the filer");
  &ttt_add_opt("filer_user", "^", "$user", "user to run as on the filer");
}

sub prereqchk { 1; }

##### Begin ttt standard preamble - do not change this code! #####

$UNIX = 1;
sub dirname
{ local($dir) = @_; $dir =~ s%^$%.%; $dir = "$dir/";
  if ($dir =~ m"^/[^/]*//*$") { return "/"; }
  if ($dir =~ m"^.*[^/]//*[^/][^/]*//*$")
    { $dir =~ s"^(.*[^/])//*[^/][^/]*//*$"$1"; { return $dir; } } #" [for cpp]
  return "."; }
$ttt_here = `/bin/pwd`; chop $ttt_here; chdir &dirname($0);
$tttroot = `/bin/pwd`; chop $tttroot; chdir $ttt_here;
require "$tttroot/tttLib.pl";

##### End ttt standard preamble - do not change the above code! #####

&ttt_rexec($srv_host, "uname -a");
&ttt_rexec($cli_host, "uname -a");

#===== record configuration information for posterity

&ttt_msg("config", "srv_host       $srv_host\n");
&ttt_msg("config", "cli_host       $cli_host\n");

#===== loopy

$vmstat_id = &ttt_start_agent($srv_host, "vmstat $vmstatint", 1, $srv_user, "vmstat");
$rbits = $rbits | $agents{$vmstat_id,"STDOUTt"} | $agents{$vmstat_id,"STDERRt"};

if ($sysstatint > 0)
  {
    $sysstat_id = &ttt_start_agent($srv_host, "rsh -l $filer_user $filer sysstat -x $sysstatint", 1, $srv_user, "sysstat");
    $rbits = $rbits | $agents{$sysstat_id,"STDOUTt"} | $agents{$sysstat_id,"STDERRt"};
  }


$last_time = 0;
$p4wd_running;

while (1)
  {
    ($line,$agentid,$stream) = &ttt_readline_agents($rbits, 1);
    if (defined ($line))
      {
        if ($line)
          {
            if ($agentid == $vmstat_id
                   || $agentid == $sysstat_id
                   || $line =~ /^exit: /
                   || $agentid == $p4wd_id
#                   || $agentid == $p4cmd_id
                                           )
              { &ttt_msg($agents{$agentid,"name"}, "$stream: $line"); }
          }
      }
    else
      {
        $rbits = $rbits ^ $agents{$agentid, $stream."t"};
        &ttt_msg($agents{$agentid,"name"}, "$stream: EOF\n");

        if (! (vec($rbits, $agents{$agentid, "STDERRf"}, 1) || vec($rbits, $agents{$agentid, "STDOUTf"}, 1)))
          {
            my $name = $agents{$agentid,"name"};
            if ($name eq "p4cmd")
              {
                $arm_p4wd = 0;
                if (&mark_time("done $p4cmd") > $p4wdthresh && ! $p4wd_running)
                  {
                     &ttt_msg("retrip", "do p4wd after probe completion\n");
                     $p4wd_id = &ttt_start_agent($srv_host, "env P4PORT=$srv_host:$srv_port p4wd 1000; top b n 1", 1, $srv_user, "p4wd");
                     $rbits = $rbits | $agents{$p4wd_id,"STDOUTt"} | $agents{$p4wd_id,"STDERRt"};
                     $p4wd_running = 1;
                  }        
              }
            elsif ($name eq "p4wd")
              { $p4wd_running = 0; }
            $status = &ttt_close_agent($agentid);
            &ttt_msg($name, "exit status = $status\n");
            if ($agentid == $p4cmd_id) { $last_time = time; }
          }
      }

    $e = tv_interval ($t0);
    if ($arm_p4wd && $last_time == -1 && ($e > $p4wdthresh) && ! $p4wd_running)
      {
        $arm_p4wd = 0;
        
        &ttt_msg("trip", "do p4wd after waiting $e seconds\n");
        $p4wd_id = &ttt_start_agent($srv_host, "env P4PORT=$srv_host:$srv_port p4wd 1000; top b n 1", 1, $srv_user, "p4wd");
        $rbits = $rbits | $agents{$p4wd_id,"STDOUTt"} | $agents{$p4wd_id,"STDERRt"};
        $p4wd_running = 1;
      }        

    if ($last_time >= 0 && ((time - $last_time) > $p4cmdint))
      {
        $p4cmd_id = &ttt_start_agent($cli_host, "p4 -p $srv_host:$srv_port $p4cmd", 1, $cli_user, "$p4cmd");
        &mark_time("start $p4cmd", "start");
        $rbits = $rbits | $agents{$p4cmd_id,"STDOUTt"} | $agents{$p4cmd_id,"STDERRt"};
        $last_time = -1;
        $arm_p4wd = 1;
      }
  }