# ============================================================================ # Copyright and license info is available in the LICENSE file included with # the Server Deployment Package (SDP), and also available online: # https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE # ---------------------------------------------------------------------------- <# .Synopsis Various SDP related shared functions .Description Functions intended to be called by other scripts. #> Function Parse-SDP-Config-File([string]$scriptpath) { $SDPConfigFile = Split-Path -parent $scriptpath | Join-Path -childpath "..\..\config\sdp_config.ini" $SDPConfigFile = Resolve-Path $SDPConfigFile $ini = Parse-Inifile($SDPConfigFile) $section = "${SDPInstance}:${env:computername}".tolower() if ($ini[$section] -eq $null) { throw("Section $section not found in ${SDPConfigFile}") } try { $global:sdp_global_root = $ini[$section]["sdp_global_root"] $global:sdp_serverid = $ini[$section]["sdp_serverid"] $global:sdp_p4serviceuser = $ini[$section]["sdp_p4serviceuser"] $global:sdp_p4superuser = $ini[$section]["sdp_p4superuser"] $global:admin_pass_filename = $ini[$section]["admin_pass_filename"] $global:mailfrom = $ini[$section]["mailfrom"] $global:maillist = $ini[$section]["maillist"] $global:mailhost = $ini[$section]["mailhost"] $global:python = $ini[$section]["python"] $global:remote_depotdata_root = $ini[$section]["remote_depotdata_root"] $result = [int32]::TryParse($ini[$section]["keepckps"], [ref]$global:keepckps) $result = [int32]::TryParse($ini[$section]["keeplogs"], [ref]$global:keeplogs) $global:limit_one_daily_checkpoint = $ini[$section]["limit_one_daily_checkpoint"] $global:remote_sdp_instance = $ini[$section]["remote_sdp_instance"] $global:p4target = $ini[$section]["p4target"] } catch { $message = $_.Exception.GetBaseException().Message Write-Error $message throw("Error parsing ini file: section ${section}") } $global:SDP_INSTANCE_HOME = -join($global:SDP_GLOBAL_ROOT, "\p4\", $SDPInstance) $global:SDP_INSTANCE_BIN_DIR = -join($global:SDP_INSTANCE_HOME, "\bin") $global:P4exe = -join($global:SDP_INSTANCE_BIN_DIR, "\p4.exe") $global:P4Dexe = -join($global:SDP_INSTANCE_BIN_DIR, "\p4d.exe") Ensure-Path-Exists $global:p4exe Ensure-Path-Exists $global:p4dexe $global:SDP_INSTANCE_SSL_DIR = -join($global:SDP_INSTANCE_HOME, "\ssl") $global:P4CONFIG = "p4config.txt" $global:P4ROOT = -join($global:SDP_INSTANCE_HOME, "\root") $global:P4LOG = -join($global:SDP_INSTANCE_HOME, "\logs\", $SDPINSTANCE, ".log") $global:P4JOURNAL = -join($global:SDP_INSTANCE_HOME, "\logs\journal") $global:P4USER = $global:SDP_P4SUPERUSER $global:P4TICKETS = -join($global:SDP_INSTANCE_BIN_DIR, "\P4Tickets.txt") $global:P4TRUST = -join($global:SDP_INSTANCE_BIN_DIR, "\P4Trust.txt") $global:P4ENVIRO = -join($global:SDP_INSTANCE_BIN_DIR, "\P4Enviro.txt") $global:SDP_INSTANCE_P4SERVICE_NAME = -join("P4_", $SdpInstance) $global:P4PORT = Get-P4PORT $global:SCRIPTS_DIR = -join($global:SDP_GLOBAL_ROOT, "\p4\common\bin") $global:LOGS_DIR = -join($global:SDP_INSTANCE_HOME, "\logs") $global:OFFLINE_DB_DIR = -join($global:SDP_INSTANCE_HOME, "\offline_db") $global:CHECKPOINTS_DIR = -join($global:SDP_INSTANCE_HOME, "\checkpoints") $global:REMOTE_CHECKPOINTS_DIR = -join($global:REMOTE_DEPOTDATA_ROOT, "\p4\", $global:REMOTE_SDP_INSTANCE, "\checkpoints") $global:ADMIN_PASS_FILE = -join($global:SCRIPTS_DIR, "\", $global:ADMIN_PASS_FILENAME) $Global:EDGE_SERVER = $false # Placeholder for now for comparison with Unix scripts # PATH=%SDP_INSTANCE_BIN_DIR%;%SCRIPTS_DIR%;%PATH% Ensure-Path-Exists $global:SDP_INSTANCE_BIN_DIR Ensure-Path-Exists $global:P4ROOT Ensure-Path-Exists $global:LOGS_DIR Ensure-Path-Exists $global:OFFLINE_DB_DIR Ensure-Path-Exists $global:CHECKPOINTS_DIR Ensure-Path-Exists $global:ADMIN_PASS_FILE } Function Get-P4PORT { $regkey = "HKLM:\SYSTEM\CurrentControlSet\services\${global:SDP_INSTANCE_P4SERVICE_NAME}\parameters" (get-itemproperty $regkey).P4PORT } Function Get-Date-Time () { Get-Date -format "yyyy\/MM\/dd HH\:mm\:ss" } Function Ensure-Path-Exists([string]$path) { if (!(Test-Path -path $path)) { throw("Error - required path $path doesn't exist!") } } Function Create-Log-File () { if (!$global:LogFileName) { throw("ERROR - Global:LogFileName not set") } $global:LogFile = Join-Path $global:LOGS_DIR $global:LogFileName write-debug "Using logfile: $global:logfile" write-output "" | set-content -path $global:Logfile } Function Log($message) { # Logs output to file and to console $datetime = get-date-time write-output "$datetime $message" write-output "$datetime $message" | add-content -path $global:Logfile } Function Parse-IniFile ($file) { $ini = @{} # Create a default section if none exist in the file. Like a java prop file. $section = "NO_SECTION" $ini[$section] = @{} switch -regex -file $file { "^\[(.+)\]$" { $section = $matches[1].Trim().ToLower() $ini[$section] = @{} } "^\s*([^#].+?)\s*=\s*(.*)" { $name,$value = $matches[1..2] # skip comments that start with semicolon: if (!($name.StartsWith(";"))) { $ini[$section][$name] = $value.Trim() } } } $ini } Function Invoke-P4Login () { Log("Logging in to P4 Instance") $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER login -a < $global:ADMIN_PASS_FILE" Log($cmd) cmd /c $cmd if (($result -eq 0) -or ($lastexitcode -ne 0)) { throw("ERROR - failed to login!") } } Function Check-Offline-DB-Exists () { Log("Checking offline DB exists") $path = Join-Path $global:OFFLINE_DB_DIR "db.counters" Log("checking existence of $path") if ( -Not (Test-Path $path -pathtype leaf)) { throw("ERROR - offline_db doesn't exist!") } } Function Ensure-Checkpoint-Not-Running () { Log("Ensuring there is no other checkpoint script running") $checkFile = Join-Path $global:CHECKPOINTS_DIR "ckp_running.txt" Log("checking existence of $checkFile") if (Test-Path $checkFile -pathtype leaf) { throw("Error - a checkpoint operation is already running!") } write-output "Checkpoint running" | set-content -path $checkFile } Function Signal-Checkpoint-Complete () { $checkFile = Join-Path $global:CHECKPOINTS_DIR "ckp_running.txt" remove-item $checkFile } Function Get-Current-Journal-Counter () { Log("Getting current live journal counter") $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER counter journal" Log($cmd) $result = Invoke-Expression $cmd Log("Current journal counter is $result") if (($result -eq 0) -or ($lastexitcode -ne 0)) { throw("Error - failed to get live journal counter!") } [int]$global:JOURNAL_NUM = $result [int]$global:CHECKPOINT_NUM = $global:JOURNAL_NUM + 1 Log("Live journal/checkpoint counters ${global:JOURNAL_NUM}/${global:CHECKPOINT_NUM}") } Function Get-Offline-Journal-Counter () { Log("Getting offline journal counter") $cmd = "$global:P4Dexe -r $global:OFFLINE_DB_DIR -jd - db.counters" Log($cmd) $rawjournal = Invoke-Expression $cmd | select-string "@journal@" | %{$_.line.split()[4]} $journal = [regex]::match($rawjournal, '^@([0-9]+)@$').Groups[1].Value if (($result -eq 0) -or ($lastexitcode -ne 0)) { throw("Error - failed to get offline journal counter!") } Log("Offline journal counter $journal") [int]$global:Offline_journal_num = $journal } Function Rotate-log-file () { param([string]$Rotate_logname, [bool]$zip) Log("Rotate-log-file $Rotate_logname, $zip") if (Test-Path $Rotate_logname -pathtype leaf) { $newname = "$Rotate_logname.$global:logid" Log("Rotating $Rotate_logname to $newname") rename-item -path $Rotate_logname -newname "$newname" if ($zip) { $gzip = -join($global:SCRIPTS_DIR, "\gzip.exe") $cmd = "$gzip $newname" Log($cmd) Invoke-Expression $cmd } } } Function Rotate-Last-Run-Logs () { Log("Rotating last run logs") $datetime = Get-Date -format "yyyyMMdd\-HHmmss" $global:LOGID = "${global:JOURNAL_NUM}.$datetime" rotate-log-file $global:LOGFILE $false rotate-log-file "${global:sdp_serverid}.log" $true rotate-log-file "log" $true rotate-log-file "p4broker.log" $true rotate-log-file "audit.log" $true rotate-log-file "sync_replica.log" $false } Function remove-files { param([string]$remove_filename, [int]$keepnum) $file_path = -join($remove_filename, "*") $files = Get-ChildItem -Path $file_path | Sort-Object -Property LastWriteTime -Descending for ($j = $keepnum; $j -lt $files.count; $j++) { $f = $files[$j].FullName Log("Removing: $f") remove-item $f -Force } } Function remove-old-logs () { # Remove old Checkpoint Logs # Use KEEPCKPS rather than KEEPLOGS, so we keep the same number # of checkpoint logs as we keep checkpoints. if ($global:keepckps -eq 0) { Log("Skipping cleanup of old checkpoint logs because KEEPCKPS is set to 0.") } else { log("Deleting old checkpoint logs. Keeping latest $global:KEEPCKPS, per KEEPCKPS setting in sdp_config.ini.") remove-files "checkpoint.log" $global:KEEPCKPS remove-files "log" $global:KEEPCKPS remove-files "p4broker.log" $global:KEEPCKPS remove-files "audit.log" $global:KEEPCKPS remove-files "sync_replica.log" $global:KEEPCKPS remove-files "recreate_offline_db.log" $global:KEEPCKPS remove-files "upgrade.log" $global:KEEPCKPS } } Function truncate-journal () { Log("Truncating live journal") $checkpoint_path = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.", $global:Checkpoint_num, ".gz") if (Test-Path $checkpoint_path -pathtype leaf) { throw("ERROR - $checkpoint_path already exists, check the backup process") } if (!$global:EDGE_SERVER) { $journalprefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME) $journalpath = -join($journalprefix, ".jnl.", $global:JOURNAL_NUM) if (Test-Path $journalpath -pathtype leaf) { throw("ERROR - $journalpath already exists, check the backup process") } log("Truncating journal...") # 'p4d -jj' does a copy-then-delete, instead of a simple mv. # during 'p4d -jj' the perforce server will hang the responses to clients. $cmd = "$global:P4Dexe -r $global:P4ROOT -J $global:P4JOURNAL -jj $journalprefix" Log($cmd) $result = Invoke-Expression $cmd Log($result) if ($lastexitcode -ne 0) { throw("ERROR - attempting to truncate journal") } } } Function replay-journals-to-offline-db () { Log("Applying all oustanding journal files to offline_db") for ($j = $global:Offline_journal_num; $j -le $global:JOURNAL_NUM; $j++) { $journalprefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME) $journalpath = -join($journalprefix, ".jnl.", $j) $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -jr $journalpath" Log($cmd) $result = Invoke-Expression $cmd Log($result) if ($lastexitcode -ne 0) { throw("ERROR - attempting to replay journal") } } } function create-offline-checkpoint { Log("Creating offline checkpoint") $checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.") if ($global:limit_one_daily_checkpoint -ne 0) { $curr_date = Get-Date -format "yyyyMMdd" $checkpoint_path = -join($checkpoint_prefix, "*.gz") $files = Get-ChildItem -Path $checkpoint_path | Sort-Object -Property LastWriteTime -Descending if ($files.length -eq 0) { Log("LIMIT_ONE_DAILY_CHECKPOINT is set to true, but no checkpoint exists.") } else { $ckp_date = $files[0].LastWriteTime | Get-Date -format "yyyyMMdd" if ($curr_date -eq $ckp_date) { Log("A checkpoint was already created today, and LIMIT_ONE_DAILY_CHECKPOINT is set to true.") Log("Skipping offline checkpoint dump.") return } } } $checkpoint_path = -join($checkpoint_prefix, $global:checkpoint_num, ".gz") $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -jd -z $checkpoint_path" Log($cmd) $result = Invoke-Expression $cmd Log($result) if ($lastexitcode -ne 0) { throw("ERROR - attempting to create offline checkpoint") } } Function recreate-offline-db-files () { Log("Recreating offline db files") $offline_db_usable = -join($global:OFFLINE_DB_DIR, "\offline_db_usable.txt") $checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.") $checkpoint_path = -join($checkpoint_prefix, "*.gz") $files = Get-ChildItem -Path $checkpoint_path | Sort-Object -Property LastWriteTime -Descending if ($files.length -eq 0) { $msg = "No checkpoints found - run live_checkpoint.sh" Log($msg) throw($msg) } if (test-path $offline_db_usable -pathtype leaf) { remove-item $offline_db_usable } $db_files = join-path $global:OFFLINE_DB_DIR "*.db" remove-item $db_files Log("Recovering from latest checkpoint found") $checkpoint_path = $files[0] $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -jr -z $checkpoint_path" Log($cmd) $result = Invoke-Expression $cmd Log($result) if ($lastexitcode -ne 0) { throw("ERROR - attempting to recover offline db files") } "Offline db file restored successfully." | set-content -path $offline_db_usable } Function remove-old-checkpoints-and-journals () { if ($global:keepckps -eq 0) { Log("Skipping cleanup of old checkpoints because KEEPCKPS is set to 0.") } else { log("Deleting obsolete checkpoints and journals. Keeping latest $global:KEEPCKPS, per KEEPCKPS setting in sdp_config.ini.") $checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.") $journal_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".jnl.") # We multiply KEEPCKP by 2 for the ckp files because of the md5 files. remove-files $checkpoint_prefix ($global:KEEPCKPS * 2) remove-files $journal_prefix ($global:KEEPCKPS) } } Function run-cmd ([string]$cmd) { Log($cmd) $result = Invoke-Expression $cmd Log($result) } Function check-disk-space () { log("Checking disk space...") $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER diskspace" run-cmd "$cmd" } Function set-counter () { Invoke-P4Login $datetime = get-date-time $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER counter lastSDPCheckpoint ""$datetime""" run-cmd -cmd $cmd } Function send-email ([string]$subject) { Log("Sending email with subject: $subject") $contents = get-content $global:logfile $SMTPServer = $global:mailhost $SMTPPort = "25" try { $message = New-Object System.Net.Mail.MailMessage $message.subject = $subject $message.body = $contents $message.to.add($global:maillist) $message.from = $global:mailfrom $smtp = New-Object System.Net.Mail.SmtpClient($SMTPServer, $SMTPPort); #$smtp.EnableSSL = $true #$smtp.Credentials = New-Object System.Net.NetworkCredential($Username, $Password); $smtp.send($message) } Catch { $ErrorMessage = $_.Exception.Message Log("Failed to send email to ${global:mailhost}: ${ErrorMessage}") } }