# ============================================================================ # Copyright and license info is available in the LICENSE file included with # the Server Deployment Package (SDP), and also available online: # https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE # ---------------------------------------------------------------------------- <# .Synopsis Various SDP related shared functions .Description Functions intended to be called by other scripts. This script is not intended to be run directly. #> Set-StrictMode -Version 2.0 Function Parse-SDPConfigFile([string]$scriptpath) { $SDPConfigFile = Split-Path -parent $scriptpath | Join-Path -childpath "..\..\config\sdp_config.ini" $SDPConfigFile = Resolve-Path $SDPConfigFile $ini = Parse-Inifile($SDPConfigFile) $section = "${SDPInstance}:${env:computername}".tolower() if ($ini[$section] -eq $null) { throw "Section $section not found in ${SDPConfigFile}" } try { $global:sdp_global_root = $ini[$section]["sdp_global_root"] $global:sdp_serverid = $ini[$section]["sdp_serverid"] $global:sdp_p4serviceuser = $ini[$section]["sdp_p4serviceuser"] $global:sdp_p4superuser = $ini[$section]["sdp_p4superuser"] $global:admin_pass_filename = $ini[$section]["admin_pass_filename"] $global:email_pass_filename = $ini[$section]["email_pass_filename"] $global:mailfrom = $ini[$section]["mailfrom"] $global:maillist = $ini[$section]["maillist"] $global:mailhost = $ini[$section]["mailhost"] $global:mailhostport = $ini[$section]["mailhostport"] $global:python = $ini[$section]["python"] $global:remote_depotdata_root = $ini[$section]["remote_depotdata_root"] $global:keepckps = 0 $global:keeplogs = 0 $result = [int32]::TryParse($ini[$section]["keepckps"], [ref]$global:keepckps) $result = [int32]::TryParse($ini[$section]["keeplogs"], [ref]$global:keeplogs) $global:limit_one_daily_checkpoint = $false $value = $ini[$section]["limit_one_daily_checkpoint"] if ($value -match "true|1|yes|y") { $global:limit_one_daily_checkpoint = $true } $global:remote_sdp_instance = $ini[$section]["remote_sdp_instance"] $global:p4target = $ini[$section]["p4target"] } catch { $message = $_.Exception.GetBaseException().Message Write-Error $message throw "Error parsing ini file: section ${section}" } $global:SDP_INSTANCE_HOME = -join($global:SDP_GLOBAL_ROOT, "\p4\", $SDPInstance) $global:SDP_INSTANCE_BIN_DIR = -join($global:SDP_INSTANCE_HOME, "\bin") $global:P4exe = -join($global:SDP_INSTANCE_BIN_DIR, "\p4.exe") $global:P4Dexe = -join($global:SDP_INSTANCE_BIN_DIR, "\p4d.exe") Ensure-PathExists $global:p4exe Ensure-PathExists $global:p4dexe $global:SDP_INSTANCE_SSL_DIR = -join($global:SDP_INSTANCE_HOME, "\ssl") $global:P4CONFIG = "p4config.txt" $global:P4ROOT = -join($global:SDP_INSTANCE_HOME, "\root") $global:P4LOG = -join($global:SDP_INSTANCE_HOME, "\logs\", $SDPINSTANCE, ".log") $global:P4JOURNAL = -join($global:SDP_INSTANCE_HOME, "\logs\journal") $global:P4USER = $global:SDP_P4SUPERUSER $env:P4TICKETS = -join($global:SDP_INSTANCE_HOME, "\P4Tickets.txt") $env:P4TRUST = -join($global:SDP_INSTANCE_HOME, "\P4Trust.txt") $env:P4ENVIRO = -join($global:SDP_INSTANCE_HOME, "\P4Enviro.txt") $global:SDP_INSTANCE_P4SERVICE_NAME = -join("p4_", $SdpInstance) $global:P4PORT = Get-P4PORT $global:SCRIPTS_DIR = -join($global:SDP_GLOBAL_ROOT, "\p4\common\bin") $global:LOGS_DIR = -join($global:SDP_INSTANCE_HOME, "\logs") $global:OFFLINE_DB_DIR = -join($global:SDP_INSTANCE_HOME, "\offline_db") $global:CHECKPOINTS_DIR = -join($global:SDP_INSTANCE_HOME, "\checkpoints") $global:REMOTE_CHECKPOINTS_DIR = -join($global:REMOTE_DEPOTDATA_ROOT, "\p4\", $global:REMOTE_SDP_INSTANCE, "\checkpoints") $global:ADMIN_PASS_FILE = -join($global:SCRIPTS_DIR, "\", $global:ADMIN_PASS_FILENAME) $global:EMAIL_PASS_FILE = -join($global:SCRIPTS_DIR, "\", $global:EMAIL_PASS_FILENAME) $Global:EDGE_SERVER = $false # Placeholder for now for comparison with Unix scripts # PATH=%SDP_INSTANCE_BIN_DIR%;%SCRIPTS_DIR%;%PATH% Ensure-PathExists $global:SDP_INSTANCE_BIN_DIR Ensure-PathExists $global:P4ROOT Ensure-PathExists $global:LOGS_DIR Ensure-PathExists $global:OFFLINE_DB_DIR Ensure-PathExists $global:CHECKPOINTS_DIR Ensure-PathExists $global:ADMIN_PASS_FILE } Function Get-P4PORT { $regkey = "HKLM:\SYSTEM\CurrentControlSet\services\${global:SDP_INSTANCE_P4SERVICE_NAME}\parameters" (get-itemproperty $regkey).P4PORT } Function Get-Date-Time () { Get-Date -format "yyyy\/MM\/dd HH\:mm\:ss" } Function Ensure-PathExists([string]$path) { if (!(Test-Path -path $path)) { throw "Error - required path $path doesn't exist!" } } Function Create-LogFile () { if (!$global:LogFileName) { throw "ERROR - Global:LogFileName not set" } $global:LogFile = Join-Path $global:LOGS_DIR $global:LogFileName Rotate-CurrentLogFile write-debug "Using logfile: $global:logfile" write-output "Starting ${global:ScriptTask}" | set-content -path $global:Logfile } Function Append-To-File([string]$message, [string]$filename) { $OFS = $null if (test-path variable:global:OFS) { $OFS = $global:OFS } $global:OFS = "`r`n" "$message" | add-content -path $filename if ($OFS) { $global:OFS = $OFS } } Function Log([string]$message) { # Logs output to file and to console $datetime = get-date-time write-host "$datetime $message" Append-To-File "$datetime $message" $global:Logfile } Function LogException([exception]$exception) { $datetime = get-date-time $message = $exception.message write-error -message "$datetime $message" -Exception $exception write-output "$datetime $message" | add-content -path $global:Logfile } Function Parse-IniFile ($file) { $ini = @{} # Create a default section if none exist in the file. Like a java prop file. $section = "NO_SECTION" $ini[$section] = @{} switch -regex -file $file { "^\[(.+)\]$" { $section = $matches[1].Trim().ToLower() $ini[$section] = @{} } "^\s*([^#].+?)\s*=\s*(.*)" { $name,$value = $matches[1..2] # skip comments that start with semicolon: if (!($name.StartsWith(";"))) { $ini[$section][$name] = $value.Trim() } } } $ini } Function Invoke-P4Login () { Log "Logging in to P4 Instance" $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER login -a < $global:ADMIN_PASS_FILE >> $global:LogFile 2>&1" Log $cmd cmd /c $cmd if ($lastexitcode -ne 0) { throw "ERROR - failed to login!" } } Function Run-ReplicaStatus ([string]$statuslog) { Log "Getting replica status" $result = & $global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER pull -lj 2>&1 Log $result if ((Test-Path $statuslog -pathtype leaf)) { remove-item $statuslog } Append-To-File "$result" $statuslog if ($lastexitcode -ne 0) { Log "WARNING - pull command exited with error code" } } Function Check-OfflineDBExists () { Log "Checking offline DB exists" $path = Join-Path $global:OFFLINE_DB_DIR "db.counters" Log "checking existence of $path" if (!(Test-Path $path -pathtype leaf)) { throw "ERROR - offline_db doesn't exist!" } } Function Ensure-CheckpointNotRunning () { Log "Ensuring there is no other checkpoint script running" $checkFile = Join-Path $global:CHECKPOINTS_DIR "ckp_running.txt" Log "checking existence of $checkFile" if (Test-Path $checkFile -pathtype leaf) { throw "Error - a checkpoint operation is already running!" } write-output "Checkpoint running" | set-content -path $checkFile } Function Signal-CheckpointComplete () { $checkFile = Join-Path $global:CHECKPOINTS_DIR "ckp_running.txt" remove-item $checkFile } Function Get-CurrentJournalCounter () { Log "Getting current live journal counter" $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER counter journal" Log $cmd $result = Invoke-Expression $cmd Log "Current journal counter is $result" if ($lastexitcode -ne 0) { throw "Error - failed to get live journal counter!" } [int]$global:JOURNAL_NUM = $result [int]$global:CHECKPOINT_NUM = $global:JOURNAL_NUM + 1 Log "Live journal/checkpoint counters ${global:JOURNAL_NUM}/${global:CHECKPOINT_NUM}" } Function Get-JournalCounter ([string]$rootdir) { $cmd = "$global:P4Dexe -r $rootdir -jd - db.counters" Log $cmd $rawjournal = Invoke-Expression $cmd | select-string "@journal@" | %{$_.line.split()[4]} Log $rawjournal $journal = [regex]::match($rawjournal, '^@([0-9]+)@$').Groups[1].Value if ($lastexitcode -ne 0) { throw "Error - failed to get $rootdir journal counter!" } return [int]$journal } Function Get-OfflineJournalCounter () { Log "Getting offline journal counter" $journal = Get-JournalCounter $global:OFFLINE_DB_DIR Log "Offline journal counter $journal" [int]$global:Offline_journal_num = $journal } Function Rotate-CurrentLogFile () { $Rotate_logname = $global:LOGFILE if (Test-Path $Rotate_logname -pathtype leaf) { $suffix = (Get-ChildItem $Rotate_logname).lastwritetime | Get-Date -format "yyyyMMdd\-HHmmss" $newname = "$Rotate_logname.$suffix" write-host "Rotating $Rotate_logname to $newname" rename-item -path $Rotate_logname -newname $newname } } Function Rotate-LogFile () { param([string]$Rotate_logname, [bool]$zip) if (Test-Path $Rotate_logname -pathtype leaf) { $newname = "$Rotate_logname.$global:logid" Log "Rotating $Rotate_logname to $newname" rename-item -path $Rotate_logname -newname "$newname" if ($zip) { $gzip = -join($global:SCRIPTS_DIR, "\gzip.exe") $cmd = "$gzip $newname" Log $cmd Invoke-Expression $cmd } } } Function Rotate-Logfiles () { Log "Rotating various logfiles" $datetime = Get-Date -format "yyyyMMdd\-HHmmss" $global:LOGID = "${global:JOURNAL_NUM}.$datetime" Rotate-LogFile "${global:sdp_serverid}.log" $true Rotate-LogFile "log" $true Rotate-LogFile "p4broker.log" $true Rotate-LogFile "audit.log" $true Rotate-LogFile "sync_replica.log" $false } Function Move-Files ([string]$source, [string]$target) { Get-ChildItem -Path $source | foreach-object { $f = $_.FullName Log "Moving $f to $target" move-item $f $target } } Function Remove-Files { # Remove older files while keeping a specified minimum number param([string]$remove_filename, [int]$keepnum) $file_path = -join($remove_filename, "*") $files = @(Get-ChildItem -Path $file_path | Sort-Object -Property LastWriteTime -Descending) if ($files) { for ($j = $keepnum; $j -lt $files.count; $j++) { $f = $files[$j].FullName Log("Removing: $f") remove-item $f -Force } } } Function Remove-OldLogs () { # Remove old Checkpoint Logs # Use KEEPCKPS rather than KEEPLOGS, so we keep the same number # of checkpoint logs as we keep checkpoints. if ($global:keepckps -eq 0) { Log "Skipping cleanup of old checkpoint logs because KEEPCKPS is set to 0." } else { log "Deleting old checkpoint logs. Keeping latest $global:KEEPCKPS, per KEEPCKPS setting in sdp_config.ini." remove-files $global:P4LOG $global:KEEPCKPS remove-files "checkpoint.log" $global:KEEPCKPS remove-files "log" $global:KEEPCKPS remove-files "p4broker.log" $global:KEEPCKPS remove-files "audit.log" $global:KEEPCKPS remove-files "sync_replica.log" $global:KEEPCKPS remove-files "recreate_offline_db.log" $global:KEEPCKPS remove-files "upgrade.log" $global:KEEPCKPS } } Function Truncate-Journal () { Log "Truncating live journal" $checkpoint_path = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.", $global:Checkpoint_num, ".gz") if (Test-Path $checkpoint_path -pathtype leaf) { throw "ERROR - $checkpoint_path already exists, check the backup process" } if (!$global:EDGE_SERVER) { $journalprefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME) $journalpath = -join($journalprefix, ".jnl.", $global:JOURNAL_NUM) if (Test-Path $journalpath -pathtype leaf) { throw "ERROR - $journalpath already exists, check the backup process" } log "Truncating journal..." # 'p4d -jj' does a copy-then-delete, instead of a simple mv. # during 'p4d -jj' the perforce server will hang the responses to clients. $cmd = "$global:P4Dexe -r $global:P4ROOT -J $global:P4JOURNAL -jj $journalprefix" Log $cmd $result = Invoke-Expression $cmd Log $result if ($lastexitcode -ne 0) { throw "ERROR - attempting to truncate journal" } } } Function Run-Checkpoint () { $checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME) $cmd = "$global:P4DEXE -r $global:P4ROOT -J $global:P4JOURNAL -jc -Z $checkpoint_prefix" Log $cmd $result = Invoke-Expression $cmd Log $result if ($lastexitcode -ne 0) { throw "ERROR - attempting to checkpoint live database" } } Function Replay-JournalsToOfflineDB () { Log "Applying all oustanding journal files to offline_db" for ($j = $global:Offline_journal_num; $j -le $global:JOURNAL_NUM; $j++) { $journalprefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME) $journalpath = -join($journalprefix, ".jnl.", $j) $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -jr $journalpath" Log $cmd $result = Invoke-Expression $cmd Log $result if ($lastexitcode -ne 0) { throw "ERROR - attempting to replay journal" } } } function Create-OfflineCheckpoint { Log "Creating offline checkpoint" $checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.") if ($global:limit_one_daily_checkpoint -ne 0) { $curr_date = Get-Date -format "yyyyMMdd" $checkpoint_path = -join($checkpoint_prefix, "*.gz") $files = @(Get-ChildItem -Path $checkpoint_path | Sort-Object -Property LastWriteTime -Descending) if (!$files -or $files.count -eq 0) { Log "LIMIT_ONE_DAILY_CHECKPOINT is set to true, but no checkpoint exists." } else { $ckp_date = $files[0].LastWriteTime | Get-Date -format "yyyyMMdd" if ($curr_date -eq $ckp_date) { Log "A checkpoint was already created today, and LIMIT_ONE_DAILY_CHECKPOINT is set to true." Log "Skipping offline checkpoint dump." return } } } $checkpoint_path = -join($checkpoint_prefix, $global:checkpoint_num, ".gz") $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -jd -z $checkpoint_path" Log $cmd $result = Invoke-Expression $cmd Log $result if ($lastexitcode -ne 0) { throw "ERROR - attempting to create offline checkpoint" } } Function Check-OfflineDBUsable () { $offline_db_usable = -join($global:OFFLINE_DB_DIR, "\offline_db_usable.txt") if (!(test-path $offline_db_usable -pathtype leaf)) { $msg = "Offline database not in a usable state. Check the backup process." Log $msg throw $msg } } Function Recreate-OfflineDBFiles () { Log "Recreating offline db files" $offline_db_usable = -join($global:OFFLINE_DB_DIR, "\offline_db_usable.txt") $checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.") $checkpoint_path = -join($checkpoint_prefix, "*.gz") $files = @(Get-ChildItem -Path $checkpoint_path | Sort-Object -Property LastWriteTime -Descending) if (!$files -or $files.count -eq 0) { $msg = "No checkpoints found - run live-checkpoint.ps1" Log $msg throw $msg } if (test-path $offline_db_usable -pathtype leaf) { remove-item $offline_db_usable } remove-files "${global:OFFLINE_DB_DIR}\db." 0 Log "Recovering from latest checkpoint found" $checkpoint_path = $files[0] $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -jr -z $checkpoint_path" Log $cmd $result = Invoke-Expression $cmd Log $result if ($lastexitcode -ne 0) { throw "ERROR - attempting to recover offline db files" } "Offline db file restored successfully." | set-content -path $offline_db_usable } Function Remove-OldCheckpointsAndJournals () { if ($global:keepckps -eq 0) { Log "Skipping cleanup of old checkpoints because KEEPCKPS is set to 0." } else { log "Deleting obsolete checkpoints and journals. Keeping latest $global:KEEPCKPS, per KEEPCKPS setting in sdp_config.ini." $checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.") $journal_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".jnl.") # We multiply KEEPCKP by 2 for the ckp files because of the md5 files. remove-files $checkpoint_prefix ($global:KEEPCKPS * 2) remove-files $journal_prefix ($global:KEEPCKPS) } } Function run-cmd ([string]$cmd) { Log $cmd $result = Invoke-Expression $cmd Log $result } Function Log-DiskSpace () { log "Checking disk space..." $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER diskspace" run-cmd "$cmd" } Function set-counter () { Invoke-P4Login $datetime = get-date-time $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER counter lastSDPCheckpoint ""$datetime""" run-cmd -cmd $cmd } Function Compare-JournalNumbers () { # Ensures offline and live db counters are the same - avoids using out-of-date offline_db $offlineJournal = Get-JournalCounter $global:OFFLINE_DB_DIR $liveJournal = Get-JournalCounter $global:P4ROOT if ($liveJournal -ne $offlineJournal) { log "$global:P4ROOT journal number is: $liveJournal" log "$global:OFFLINE_DB_DIR journal number is: $offlineJournal" throw "$global:P4ROOT and $global:OFFLINE_DB_DIR journal numbers do not match." } } Function Move-OfflineDBToLive () { # Compare the Offline and Master journal numbers before switching to make sure they match. Compare-JournalNumbers log "Switching out db files..." $savedir = "${global:P4ROOT}\save" if (!(test-path -path $savedir -type container)) { New-Item $savedir -ItemType directory } remove-files "$savedir\db." 0 move-files "${global:P4ROOT}\db.*" $savedir move-files "${global:offline_db_dir}\db.*" $global:P4ROOT remove-item "${global:offline_db_dir}\offline_db_usable.txt" } Function Test-IsAdmin { ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator") } Function Check-AdminPrivileges { if (!(Test-IsAdmin)) { throw "Please run this script with admin priviliges" } } Function Stop-LiveService () { Stop-Service $global:SDP_INSTANCE_P4SERVICE_NAME } Function Start-LiveService () { Start-Service $global:SDP_INSTANCE_P4SERVICE_NAME } Function send-email ([string]$subject, [string]$logfilename) { Log "Sending email with subject: $subject" if ($logfilename) { $contents = get-content $logfilename } else { $contents = get-content $global:logfile } $SMTPServer = $global:mailhost $SMTPPort = "25" if ($global:mailhostport -ne "") { $SMTPPort = $global:mailhostport } try { $email_password = "" if ((test-path $global:email_pass_file -pathtype leaf)) { Log "Found $global:email_pass_file" $email_password = get-content $global:email_pass_file } $message = New-Object System.Net.Mail.MailMessage $message.subject = $subject $message.body = $contents $message.to.add($global:maillist) $message.from = $global:mailfrom $smtp = New-Object System.Net.Mail.SmtpClient($SMTPServer, $SMTPPort) if ($SMTPPort -ne "25") { $smtp.EnableSSL = $true } if ($email_password -ne "") { $smtp.Credentials = New-Object System.Net.NetworkCredential($global:mailfrom, $email_password) } $smtp.send($message) } Catch { $ErrorMessage = $_.Exception.Message Log "Failed to send email to ${global:mailhost}: ${ErrorMessage}" } }