parse_sysctl.pl #1

#!/usr/bin/perl
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

# Author: jhalbig@perforce.com
# Date: November 10, 2020

# After several years of successful implementation my plan is to render this 
# ready for prime-time usage by cleaning up and fully documenting the settings 
# with an internal KB that will one day be presentable to a wider audience; for
# now this will be useful for support engineers for cases where the server loads
# don't appear to be utilizing all Linux OS hardware, networking resources in
# particular.
#
# Usage: ./parse_sysctl.pl {-a} {/path to/}sysctl.out
#
#     -a: 
#
#     Check rarely tweaked kernel settings.
#
#     {/path to/}sysctl.out:
#
#     The 'sysctl -a' output file to parse.
#
# Or change this sysctl_file value to a file containing "sysctl -a" output:

my $sysctl_file = "sysctl.out";

# List of recommended kernel values:
my $rec_k_vals = <<KERNEL_VALS;
	net.ipv4.tcp_syncookies					: 0
	net.ipv4.tcp_timestamps					: 1
	net.ipv4.tcp_window_scaling				: 1
	net.core.somaxconn						: 2048
	net.core.netdev_max_backlog				: 5000
	net.ipv4.tcp_max_syn_backlog 			: 2048
	net.ipv4.ip_local_port_range 			: 10000 65535
	net.ipv4.tcp_fin_timeout 				: 30
	net.ipv4.tcp_keepalive_intvl 			: 30
	net.ipv4.tcp_keepalive_probes 			: 20
	net.ipv4.tcp_keepalive_time 			: 890
	net.core.netdev_budget					: 600
	net.core.rmem_max 						: 16777216
	net.core.wmem_max 						: 16777216
	net.core.optmem_max						: 16777216
	net.ipv4.tcp_mem 						: 1528512 2038016 8388608
	net.ipv4.tcp_rmem 						: 4096 87380 16777216
	net.ipv4.tcp_wmem 						: 4096 65536 16777216
	net.ipv4.tcp_no_metrics_save			: 1
	net.ipv4.tcp_sack						: 0
	net.ipv4.conf.all.accept_redirects		: 0
	net.ipv4.conf.all.rp_filter				: 1
	net.ipv4.conf.all.secure_redirects		: 0
	net.ipv4.conf.all.send_redirects		: 0
	net.ipv4.conf.all.accept_source_route	: 0
	net.ipv4.icmp_echo_ignore_broadcasts	: 1
	net.ipv4.ip_forward						: 0
KERNEL_VALS

my $rare_k_vals = <<KERNEL_VALS;
	kernel.nmi_watchdog						: 0
	kernel.soft_watchdog					: 1
	kernel.watchdog_thresh					: 180
	kernel.softlockup_all_cpu_backtrace		: 1
	kernel.hardlockup_all_cpu_backtrace		: 0
	kernel.sysrq							: 0
	kernel.core_uses_pid					: 1
	kernel.msgmnb							: 65536
	kernel.msgmax							: 65536
	kernel.shmmax							: 68719476736
	kernel.shmall							: 4294967296
KERNEL_VALS

# OS command to change/write values:
my $sysctl_cmd = "sysctl -w";

# List of recommended Perforce server configurables:
my %p4d_vals = qw(
	net.tcpsize 524288
	net.backlog 2048
	filesys.bufsize 512K
	net.bufsize 512K
	filesys.bufsize 512K
	lbr.bufsize 512K
);

# Perforce command to change/write values:
my $p4_cmd = "p4 configure set";

# Those are the text sections that allow the Perforce VCS admin to adapt this  
# script to their particular needs:
my $txt_1 = <<PREAMBLE_TEXT;

Some key performance related kernel settings are at the defaults, which means that some of your server resources are likely under utilized. Here is a list of the settings that, based on past experience, improve server performance, in some cases dramatically:

	Setting		New Value	(Old Value)
	---------------------------------------
PREAMBLE_TEXT

my $txt_2 = <<CHG_KERNEL_HEADER;

To change those settings, run those commands as a privileged user:

CHG_KERNEL_HEADER

my $txt_3 = <<UPDATESYSCTL_TEXT;

Note: Always make a copy of your sysctl.conf file prior to making any changes, whether by editing the file directly or using the sysctl command:
 
	sudo mv /etc/sysctl.conf /etc/sysctl.conf.BAK

Once you've backed up this file and determined that there are no issues with the new settings, make those changes to the same values listed in that file.

To test the new sysctl.conf file, use the command:

	sudo sysctl -p

This will re-load all kernel settings directly from sysctl.conf file. 

Important note: If things appear "broken" after this point, *do not reboot*; Make a copy of the edited sysctl.conf file and replace it from the backup you created earlier, and reload the file using the same command.

UPDATESYSCTL_TEXT

my $txt_4 = <<CHG_P4_HEADER;

Next, run those commands against your Perforce server:

CHG_P4_HEADER

my $txt_5 = <<P4_RESTART;
	p4 admin restart
P4_RESTART

my $txt_6 = <<CLOSING_TEXT;

The last command is needed for the new values to take effect for the Perforce server as several of those configurables are not dynamically change upon setting.

CLOSING_TEXT

my $usage_txt = <<USAGE; 
Usage: 
	./parse_sysctl.pl {-a} {/path to/}sysctl.out

Arguments:
    -a: 

      Check rarely tweaked kernel settings.

    -h/-?
    
      This message.

    {/path to/}sysctl.out:

      The 'sysctl -a' output file to parse.

Note:
	If no file argument is given the script looks for "$sysctl_file" by default.
	Modify the sysctl_file variable in the script to change this.

For more information, see:

	Using parse_sysctl.pl Script To Diagnose Linux Performance Issues
	https://perforce.my.salesforce.com/kA02I000000bn4u
	
USAGE

## End user settings.

# Check for -a option:
if ($ARGV[0] =~ "-a") { 
	shift; 
	$rec_k_vals .= $rare_k_vals;
} elsif ($ARGV[0] =~ m/-(h|\?)/) {
	shift; 
	print $usage_txt;
	exit(0);
}

# Check for file argument:
if ($ARGV[0]){ $sysctl_file = $ARGV[0]; }

my %k_vals;
my $q = "";
my $a = 0;
@l = ("aa" .. "zz");

# I wanted to keep kernel settings groups as they are to keep related settings
# grouped together. After considerable mucking about it was clear that sorting 
# a hash based on order of entry was not possible without adding an index. 
#
# Which was promptly sorted in that annoying "1,11,12,2,3..." pattern. I REALLY
# just wanted to sort it in order, so I created an array of labels from aa through zz,
# Which covers 676 possible entries. Likely overkill. :-) However, if it's not enough I
# can always add a letter. :P
#
# Just know I won't be the one populating that many kernel tweaks. ;-)
#
# The upshot is I can preserve the original order by sorting the hash keys, instead 
# of iterating through arrays and messing with hash slices. W00t!
#
# I parse the text block to create a hash to track kernel settings:
while ($rec_k_vals =~ m/.+?(.+?)\s*\:\s*(.+)\s*?\n/g) {
	$k_vals{$l[$a]} = {'name' => $1, 'old_val' => "none", 'new_val' => $2};
	$a++;
}

# Parse the sysctl output file to determine if there are any values that need to
# be changed or added:
open (SYSCTL, "< " . $sysctl_file) 
	or die $usage_txt . "ERROR: Can't read file " . $sysctl_file .":\n" . $!;
while (<SYSCTL>) {
	chomp;
	foreach my $val (keys %k_vals) {
		if (/^.*?$k_vals{$val}{'name'}\s+?=\s+?(.*)$/) { $k_vals{$val}{'old_val'} = $1; }
	}
}
close SYSCTL;

## Create the pretty table with comparison values
print $txt_1;

foreach my $val (sort keys %k_vals) {
	$k_vals{$val}{'old_val'} =~ s/\t/\x20/g; # normalize tabs to spaces
	unless ($k_vals{$val}{'old_val'} =~ $k_vals{$val}{'new_val'}) {
		if ($k_vals{$val}{'old_val'}  =~ "none") { next; }
		if ( $k_vals{$val}{old_val} == 0 || $k_vals{$val}{old_val } ) {
			print "\t" . $k_vals{$val}{'name'} . "\t\t" . 
			$k_vals{$val}{new_val} . "\t" . "(" . $k_vals{$val}{old_val} . ")\n";
	    }
	} 
}

## Create the ready to paste OS commands to change those settings:
print $txt_2;

foreach my $val (sort keys %k_vals) {
	unless ($k_vals{$val}{old_val} =~ $k_vals{$val}{new_val}) {
				if ($k_vals{$val}{'old_val'}  =~ "none") { next; }

	    unless ($k_vals{$val}{old_val}) { next; }
		if ($k_vals{$val}{new_val} =~ m/\s/g) { $q = "\""; }
		print "\t" . $sysctl_cmd . " " . $k_vals{$val}{'name'} . "=" . $q . 
		$k_vals{$val}{new_val} . $q . "\n";
		$q = "";
	}
}

print $txt_3;
print $txt_4;

## Create the ready to paste Perforce server commands:
foreach my $val (keys %p4d_vals) {
	print "\t" . $p4_cmd . " " . $val . "=" . $p4d_vals{$val} . "\n";
}

## Print "p4 admin restart" command:
print $txt_5;

## Closing text.
print $txt_6;

# WARNING: While this detects differences in values, the original values might
# already have been tweaked! Remember to remove those lines that are already
# at good values, taking care to remove the associated "sysctl -w" entry.