# ============================================================================ # Copyright and license info is available in the LICENSE file included with # the Server Deployment Package (SDP), and also available online: # https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE # ---------------------------------------------------------------------------- <# .Synopsis Various SDP related shared functions .Description Functions intended to be called by other scripts. #> Function Parse-SDPConfigFile([string]$scriptpath) { $SDPConfigFile = Split-Path -parent $scriptpath | Join-Path -childpath "..\..\config\sdp_config.ini" $SDPConfigFile = Resolve-Path $SDPConfigFile $ini = Parse-Inifile($SDPConfigFile) $section = "${SDPInstance}:${env:computername}".tolower() if ($ini[$section] -eq $null) { throw("Section $section not found in ${SDPConfigFile}") } try { $global:sdp_global_root = $ini[$section]["sdp_global_root"] $global:sdp_serverid = $ini[$section]["sdp_serverid"] $global:sdp_p4serviceuser = $ini[$section]["sdp_p4serviceuser"] $global:sdp_p4superuser = $ini[$section]["sdp_p4superuser"] $global:admin_pass_filename = $ini[$section]["admin_pass_filename"] $global:mailfrom = $ini[$section]["mailfrom"] $global:maillist = $ini[$section]["maillist"] $global:mailhost = $ini[$section]["mailhost"] $global:python = $ini[$section]["python"] $global:remote_depotdata_root = $ini[$section]["remote_depotdata_root"] $result = [int32]::TryParse($ini[$section]["keepckps"], [ref]$global:keepckps) $result = [int32]::TryParse($ini[$section]["keeplogs"], [ref]$global:keeplogs) $global:limit_one_daily_checkpoint = $ini[$section]["limit_one_daily_checkpoint"] $global:remote_sdp_instance = $ini[$section]["remote_sdp_instance"] $global:p4target = $ini[$section]["p4target"] } catch { $message = $_.Exception.GetBaseException().Message Write-Error $message throw("Error parsing ini file: section ${section}") } $global:SDP_INSTANCE_HOME = -join($global:SDP_GLOBAL_ROOT, "\p4\", $SDPInstance) $global:SDP_INSTANCE_BIN_DIR = -join($global:SDP_INSTANCE_HOME, "\bin") $global:P4exe = -join($global:SDP_INSTANCE_BIN_DIR, "\p4.exe") $global:P4Dexe = -join($global:SDP_INSTANCE_BIN_DIR, "\p4d.exe") Ensure-PathExists $global:p4exe Ensure-PathExists $global:p4dexe $global:SDP_INSTANCE_SSL_DIR = -join($global:SDP_INSTANCE_HOME, "\ssl") $global:P4CONFIG = "p4config.txt" $global:P4ROOT = -join($global:SDP_INSTANCE_HOME, "\root") $global:P4LOG = -join($global:SDP_INSTANCE_HOME, "\logs\", $SDPINSTANCE, ".log") $global:P4JOURNAL = -join($global:SDP_INSTANCE_HOME, "\logs\journal") $global:P4USER = $global:SDP_P4SUPERUSER $global:P4TICKETS = -join($global:SDP_INSTANCE_BIN_DIR, "\P4Tickets.txt") $global:P4TRUST = -join($global:SDP_INSTANCE_BIN_DIR, "\P4Trust.txt") $global:P4ENVIRO = -join($global:SDP_INSTANCE_BIN_DIR, "\P4Enviro.txt") $global:SDP_INSTANCE_P4SERVICE_NAME = -join("P4_", $SdpInstance) $global:P4PORT = Get-P4PORT $global:SCRIPTS_DIR = -join($global:SDP_GLOBAL_ROOT, "\p4\common\bin") $global:LOGS_DIR = -join($global:SDP_INSTANCE_HOME, "\logs") $global:OFFLINE_DB_DIR = -join($global:SDP_INSTANCE_HOME, "\offline_db") $global:CHECKPOINTS_DIR = -join($global:SDP_INSTANCE_HOME, "\checkpoints") $global:REMOTE_CHECKPOINTS_DIR = -join($global:REMOTE_DEPOTDATA_ROOT, "\p4\", $global:REMOTE_SDP_INSTANCE, "\checkpoints") $global:ADMIN_PASS_FILE = -join($global:SCRIPTS_DIR, "\", $global:ADMIN_PASS_FILENAME) $Global:EDGE_SERVER = $false # Placeholder for now for comparison with Unix scripts # PATH=%SDP_INSTANCE_BIN_DIR%;%SCRIPTS_DIR%;%PATH% Ensure-PathExists $global:SDP_INSTANCE_BIN_DIR Ensure-PathExists $global:P4ROOT Ensure-PathExists $global:LOGS_DIR Ensure-PathExists $global:OFFLINE_DB_DIR Ensure-PathExists $global:CHECKPOINTS_DIR Ensure-PathExists $global:ADMIN_PASS_FILE } Function Get-P4PORT { $regkey = "HKLM:\SYSTEM\CurrentControlSet\services\${global:SDP_INSTANCE_P4SERVICE_NAME}\parameters" (get-itemproperty $regkey).P4PORT } Function Get-Date-Time () { Get-Date -format "yyyy\/MM\/dd HH\:mm\:ss" } Function Ensure-PathExists([string]$path) { if (!(Test-Path -path $path)) { throw("Error - required path $path doesn't exist!") } } Function Create-LogFile () { if (!$global:LogFileName) { throw("ERROR - Global:LogFileName not set") } $global:LogFile = Join-Path $global:LOGS_DIR $global:LogFileName Rotate-CurrentLogFile write-debug "Using logfile: $global:logfile" write-output "Starting ${global:ScriptTask}" | set-content -path $global:Logfile } Function Log($message) { # Logs output to file and to console $datetime = get-date-time write-output "$datetime $message" write-output "$datetime $message" | add-content -path $global:Logfile } Function Parse-IniFile ($file) { $ini = @{} # Create a default section if none exist in the file. Like a java prop file. $section = "NO_SECTION" $ini[$section] = @{} switch -regex -file $file { "^\[(.+)\]$" { $section = $matches[1].Trim().ToLower() $ini[$section] = @{} } "^\s*([^#].+?)\s*=\s*(.*)" { $name,$value = $matches[1..2] # skip comments that start with semicolon: if (!($name.StartsWith(";"))) { $ini[$section][$name] = $value.Trim() } } } $ini } Function Invoke-P4Login () { Log("Logging in to P4 Instance") $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER login -a < $global:ADMIN_PASS_FILE" Log($cmd) cmd /c $cmd if (($result -eq 0) -or ($lastexitcode -ne 0)) { throw("ERROR - failed to login!") } } Function Check-OfflineDBExists () { Log("Checking offline DB exists") $path = Join-Path $global:OFFLINE_DB_DIR "db.counters" Log("checking existence of $path") if ( -Not (Test-Path $path -pathtype leaf)) { throw("ERROR - offline_db doesn't exist!") } } Function Ensure-CheckpointNotRunning () { Log("Ensuring there is no other checkpoint script running") $checkFile = Join-Path $global:CHECKPOINTS_DIR "ckp_running.txt" Log("checking existence of $checkFile") if (Test-Path $checkFile -pathtype leaf) { throw("Error - a checkpoint operation is already running!") } write-output "Checkpoint running" | set-content -path $checkFile } Function Signal-CheckpointComplete () { $checkFile = Join-Path $global:CHECKPOINTS_DIR "ckp_running.txt" remove-item $checkFile } Function Get-CurrentJournalCounter () { Log("Getting current live journal counter") $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER counter journal" Log($cmd) $result = Invoke-Expression $cmd Log("Current journal counter is $result") if (($result -eq 0) -or ($lastexitcode -ne 0)) { throw("Error - failed to get live journal counter!") } [int]$global:JOURNAL_NUM = $result [int]$global:CHECKPOINT_NUM = $global:JOURNAL_NUM + 1 Log("Live journal/checkpoint counters ${global:JOURNAL_NUM}/${global:CHECKPOINT_NUM}") } Function Get-OfflineJournalCounter () { Log("Getting offline journal counter") $cmd = "$global:P4Dexe -r $global:OFFLINE_DB_DIR -jd - db.counters" Log($cmd) $rawjournal = Invoke-Expression $cmd | select-string "@journal@" | %{$_.line.split()[4]} $journal = [regex]::match($rawjournal, '^@([0-9]+)@$').Groups[1].Value if (($result -eq 0) -or ($lastexitcode -ne 0)) { throw("Error - failed to get offline journal counter!") } Log("Offline journal counter $journal") [int]$global:Offline_journal_num = $journal } Function Rotate-CurrentLogFile () { $Rotate_logname = $global:LOGFILE if (Test-Path $Rotate_logname -pathtype leaf) { $suffix = (Get-ChildItem $Rotate_logname).lastwritetime | Get-Date -format "yyyyMMdd\-HHmmss" $newname = "$Rotate_logname.$suffix" write-host "Rotating $Rotate_logname to $newname" rename-item -path $Rotate_logname -newname $newname } } Function Rotate-LogFile () { param([string]$Rotate_logname, [bool]$zip) if (Test-Path $Rotate_logname -pathtype leaf) { $newname = "$Rotate_logname.$global:logid" Log("Rotating $Rotate_logname to $newname") rename-item -path $Rotate_logname -newname "$newname" if ($zip) { $gzip = -join($global:SCRIPTS_DIR, "\gzip.exe") $cmd = "$gzip $newname" Log($cmd) Invoke-Expression $cmd } } } Function Rotate-Logfiles () { Log("Rotating various logfiless") $datetime = Get-Date -format "yyyyMMdd\-HHmmss" $global:LOGID = "${global:JOURNAL_NUM}.$datetime" Rotate-LogFile "${global:sdp_serverid}.log" $true Rotate-LogFile "log" $true Rotate-LogFile "p4broker.log" $true Rotate-LogFile "audit.log" $true Rotate-LogFile "sync_replica.log" $false } Function Remove-Files { param([string]$remove_filename, [int]$keepnum) $file_path = -join($remove_filename, "*") $files = Get-ChildItem -Path $file_path | Sort-Object -Property LastWriteTime -Descending for ($j = $keepnum; $j -lt $files.count; $j++) { $f = $files[$j].FullName Log("Removing: $f") remove-item $f -Force } } Function Remove-OldLogs () { # Remove old Checkpoint Logs # Use KEEPCKPS rather than KEEPLOGS, so we keep the same number # of checkpoint logs as we keep checkpoints. if ($global:keepckps -eq 0) { Log("Skipping cleanup of old checkpoint logs because KEEPCKPS is set to 0.") } else { log("Deleting old checkpoint logs. Keeping latest $global:KEEPCKPS, per KEEPCKPS setting in sdp_config.ini.") remove-files "checkpoint.log" $global:KEEPCKPS remove-files "log" $global:KEEPCKPS remove-files "p4broker.log" $global:KEEPCKPS remove-files "audit.log" $global:KEEPCKPS remove-files "sync_replica.log" $global:KEEPCKPS remove-files "recreate_offline_db.log" $global:KEEPCKPS remove-files "upgrade.log" $global:KEEPCKPS } } Function Truncate-Journal () { Log("Truncating live journal") $checkpoint_path = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.", $global:Checkpoint_num, ".gz") if (Test-Path $checkpoint_path -pathtype leaf) { throw("ERROR - $checkpoint_path already exists, check the backup process") } if (!$global:EDGE_SERVER) { $journalprefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME) $journalpath = -join($journalprefix, ".jnl.", $global:JOURNAL_NUM) if (Test-Path $journalpath -pathtype leaf) { throw("ERROR - $journalpath already exists, check the backup process") } log("Truncating journal...") # 'p4d -jj' does a copy-then-delete, instead of a simple mv. # during 'p4d -jj' the perforce server will hang the responses to clients. $cmd = "$global:P4Dexe -r $global:P4ROOT -J $global:P4JOURNAL -jj $journalprefix" Log($cmd) $result = Invoke-Expression $cmd Log($result) if ($lastexitcode -ne 0) { throw("ERROR - attempting to truncate journal") } } } Function Replay-JournalsToOfflineDB () { Log("Applying all oustanding journal files to offline_db") for ($j = $global:Offline_journal_num; $j -le $global:JOURNAL_NUM; $j++) { $journalprefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME) $journalpath = -join($journalprefix, ".jnl.", $j) $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -jr $journalpath" Log($cmd) $result = Invoke-Expression $cmd Log($result) if ($lastexitcode -ne 0) { throw("ERROR - attempting to replay journal") } } } function Create-OfflineCheckpoint { Log("Creating offline checkpoint") $checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.") if ($global:limit_one_daily_checkpoint -ne 0) { $curr_date = Get-Date -format "yyyyMMdd" $checkpoint_path = -join($checkpoint_prefix, "*.gz") $files = Get-ChildItem -Path $checkpoint_path | Sort-Object -Property LastWriteTime -Descending if ($files.length -eq 0) { Log("LIMIT_ONE_DAILY_CHECKPOINT is set to true, but no checkpoint exists.") } else { $ckp_date = $files[0].LastWriteTime | Get-Date -format "yyyyMMdd" if ($curr_date -eq $ckp_date) { Log("A checkpoint was already created today, and LIMIT_ONE_DAILY_CHECKPOINT is set to true.") Log("Skipping offline checkpoint dump.") return } } } $checkpoint_path = -join($checkpoint_prefix, $global:checkpoint_num, ".gz") $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -jd -z $checkpoint_path" Log($cmd) $result = Invoke-Expression $cmd Log($result) if ($lastexitcode -ne 0) { throw("ERROR - attempting to create offline checkpoint") } } Function Recreate-OfflineDBFiles () { Log("Recreating offline db files") $offline_db_usable = -join($global:OFFLINE_DB_DIR, "\offline_db_usable.txt") $checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.") $checkpoint_path = -join($checkpoint_prefix, "*.gz") $files = Get-ChildItem -Path $checkpoint_path | Sort-Object -Property LastWriteTime -Descending if ($files.length -eq 0) { $msg = "No checkpoints found - run live_checkpoint.sh" Log($msg) throw($msg) } if (test-path $offline_db_usable -pathtype leaf) { remove-item $offline_db_usable } $db_files = join-path $global:OFFLINE_DB_DIR "*.db" remove-item $db_files Log("Recovering from latest checkpoint found") $checkpoint_path = $files[0] $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -jr -z $checkpoint_path" Log($cmd) $result = Invoke-Expression $cmd Log($result) if ($lastexitcode -ne 0) { throw("ERROR - attempting to recover offline db files") } "Offline db file restored successfully." | set-content -path $offline_db_usable } Function Remove-OldCheckpointsAndJournals () { if ($global:keepckps -eq 0) { Log("Skipping cleanup of old checkpoints because KEEPCKPS is set to 0.") } else { log("Deleting obsolete checkpoints and journals. Keeping latest $global:KEEPCKPS, per KEEPCKPS setting in sdp_config.ini.") $checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.") $journal_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".jnl.") # We multiply KEEPCKP by 2 for the ckp files because of the md5 files. remove-files $checkpoint_prefix ($global:KEEPCKPS * 2) remove-files $journal_prefix ($global:KEEPCKPS) } } Function run-cmd ([string]$cmd) { Log($cmd) $result = Invoke-Expression $cmd Log($result) } Function Log-DiskSpace () { log("Checking disk space...") $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER diskspace" run-cmd "$cmd" } Function set-counter () { Invoke-P4Login $datetime = get-date-time $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER counter lastSDPCheckpoint ""$datetime""" run-cmd -cmd $cmd } Function send-email ([string]$subject) { Log("Sending email with subject: $subject") $contents = get-content $global:logfile $SMTPServer = $global:mailhost $SMTPPort = "25" try { $message = New-Object System.Net.Mail.MailMessage $message.subject = $subject $message.body = $contents $message.to.add($global:maillist) $message.from = $global:mailfrom $smtp = New-Object System.Net.Mail.SmtpClient($SMTPServer, $SMTPPort); #$smtp.EnableSSL = $true #$smtp.Credentials = New-Object System.Net.NetworkCredential($Username, $Password); $smtp.send($message) } Catch { $ErrorMessage = $_.Exception.Message Log("Failed to send email to ${global:mailhost}: ${ErrorMessage}") } }
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#39 | 30906 | Robert Cowham | Fix SDP-1137 issue with NativeCommandError (somewhat sprious error but worrying in logs!) | ||
#38 | 30353 | Will Kreitzmann | Edge checkpoints | ||
#37 | 28771 | C. Thomas Tyler |
Changed email address for Perforce Support. #review-28772 @amo @robert_cowham |
||
#36 | 28089 | Robert Cowham | User serverid for p4log filename. | ||
#35 | 27915 | Robert Cowham |
Implement p4login.bat properly with powershell. Fix to @27817 #review ttyler |
||
#34 | 27722 | C. Thomas Tyler |
Refinements to @27712: * Resolved one out-of-date file (verify_sdp.sh). * Added missing adoc file for which HTML file had a change (WorkflowEnforcementTriggers.adoc). * Updated revdate/revnumber in *.adoc files. * Additional content updates in Server/Unix/p4/common/etc/cron.d/ReadMe.md. * Bumped version numbers on scripts with Version= def'n. * Generated HTML, PDF, and doc/gen files: - Most HTML and all PDF are generated using Makefiles that call an AsciiDoc utility. - HTML for Perl scripts is generated with pod2html. - doc/gen/*.man.txt files are generated with .../tools/gen_script_man_pages.sh. #review-27712 |
||
#33 | 26658 | Robert Cowham | Make this file keytext for versioning | ||
#32 | 26528 | Robert Cowham |
Start/stop services with svcinst now Refactor run-cmd to log command output for ease of support, e.g. when daily-checkpoint fails. |
||
#31 | 26155 | C. Thomas Tyler |
Added '-f' flag to 'p4d -jr' command, for Windows SDP, to match similar change made previously in the Linux SDP. |
||
#30 | 26151 | Robert Cowham | Make sure the rename works as expected and retries - was previously still not doing so. | ||
#29 | 26120 | Robert Cowham |
Fix daily_checkpoint for edge/standby servers Make edge tables to include/exclude version specific Move password files to config dir |
||
#28 | 26106 | Robert Cowham | Correct log message | ||
#27 | 26104 | Robert Cowham | Add a retry loop for log renames (in case of server log being locked by busy server) | ||
#26 | 26054 | Robert Cowham | Fix log typo | ||
#25 | 25544 | C. Thomas Tyler |
Removed journalPrefix as command line paramter during journal rotation, deferring to db.config values, for Windows SDP. Removed explicit specification of journalPrefix as a command line argument to 'p4d -jj' command. Specifying the prefix is redundant as journalPrefix values are defined for the master server and any/all replicas/edge servers in db.config, and db.config is the One Source of Truth for journalPrefix. #review @robert_cowham @mshields |
||
#24 | 22984 | Robert Cowham | Make sure rotate works | ||
#23 | 22922 | Robert Cowham |
Add a utility to rotate log files - good as a scheduled task for replicas who may not have other jobs scheduled. |
||
#22 | 22919 | Robert Cowham |
Refacto sdp-functions Convert upgrade.bat into upgrade.ps1 Fix issue with p4verify.ps1 |
||
#21 | 22918 | Robert Cowham | Improve error message | ||
#20 | 22809 | Robert Cowham | Fix problem when mailhostport not set in sdp_config.ini - now defaults properly to 25. | ||
#19 | 22725 | Robert Cowham |
Remove unused parameter. Cope with multiple runs. |
||
#18 | 22723 | Robert Cowham | New function used by recover-edge.ps1 | ||
#17 | 20885 | Robert Cowham | Fix formatting of logs - when sent by gmail especially - use HTML with simple line breaks. | ||
#16 | 20646 | Robert Cowham |
Got it working as far daily_backup tests. Refactor a few names. Auto wrap .ps1 commands. |
||
#15 | 20624 | Robert Cowham |
Handle global:OFS properly if not set. Useful for array output. |
||
#14 | 20595 | Robert Cowham |
New replica-status command Add corresponding function Improve end of line handling when logging |
||
#13 | 20545 | Robert Cowham | Updated for sending emails via gmail accounts - known to work (if properly configured!) | ||
#12 | 20537 | Robert Cowham | Update send-email to work with gmail accounts | ||
#11 | 20296 | Robert Cowham | Avoid using write-output in log function so we don't have to be so careful with returns from get-journalCounter | ||
#10 | 20293 | Robert Cowham |
Implement recreate-live-from-offline_db.ps1 - replacement fro weekly b_backup.bat which no longer needs to be run weekly. |
||
#9 | 20204 | Robert Cowham | Fix checkpoint error - make sure we rotate correct journal. | ||
#8 | 20186 | Robert Cowham |
Fix typo. Tidy up old server logs. |
||
#7 | 20184 | Robert Cowham |
Added live-checkpoint.ps1 Fixed problem where all log files and others being removed - not keeping required number. |
||
#6 | 20175 | Robert Cowham |
Set-strictmode Remove warnings Improve exception logging |
||
#5 | 20150 | Robert Cowham | Refactored names to use PowerShell Verb-Noun convention | ||
#4 | 20149 | Robert Cowham | Split rotation of current script log file from rotation of other log files. | ||
#3 | 20148 | Robert Cowham | Added log info for functions | ||
#2 | 20146 | Robert Cowham |
Refactor email sending. Output final message of success/failure. |
||
#1 | 20142 | Robert Cowham | Initial versions of Powershell scripts |