SDP-functions.ps1 #2

# ============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
# ----------------------------------------------------------------------------

<#
    .Synopsis
        Various SDP related shared functions
        
    .Description
        Functions intended to be called by other scripts.
        This script is not intended to be run directly.
        
#>

Set-StrictMode -Version 2.0

Function Parse-SDPConfigFile([string]$scriptpath) {
    $SDPConfigFile = Split-Path -parent $scriptpath | Join-Path -childpath "..\..\config\sdp_config.ini"
    $SDPConfigFile = Resolve-Path $SDPConfigFile
    $ini = Parse-Inifile($SDPConfigFile)

    $section = "${SDPInstance}:${env:computername}".tolower()
    if ($ini[$section] -eq $null) {
        throw "Section $section not found in ${SDPConfigFile}"
    }
    
    try {
        $global:sdp_global_root = $ini[$section]["sdp_global_root"]
        $global:sdp_serverid = $ini[$section]["sdp_serverid"]
        $global:sdp_p4serviceuser = $ini[$section]["sdp_p4serviceuser"]
        $global:sdp_p4superuser = $ini[$section]["sdp_p4superuser"]
        $global:admin_pass_filename = $ini[$section]["admin_pass_filename"]
        $global:email_pass_filename = $ini[$section]["email_pass_filename"]
        $global:mailfrom = $ini[$section]["mailfrom"]
        $global:maillist = $ini[$section]["maillist"]
        $global:mailhost = $ini[$section]["mailhost"]
        $global:mailhostport = $ini[$section]["mailhostport"]
        $global:python = $ini[$section]["python"]
        $global:remote_depotdata_root = $ini[$section]["remote_depotdata_root"]
        $global:keepckps = 0
        $global:keeplogs = 0
        $result = [int32]::TryParse($ini[$section]["keepckps"], [ref]$global:keepckps)
        $result = [int32]::TryParse($ini[$section]["keeplogs"], [ref]$global:keeplogs)
        $global:limit_one_daily_checkpoint = $false
        $value = $ini[$section]["limit_one_daily_checkpoint"]
        if ($value -match "true|1|yes|y") {
            $global:limit_one_daily_checkpoint = $true
        }
        $global:remote_sdp_instance = $ini[$section]["remote_sdp_instance"]
        $global:p4target = $ini[$section]["p4target"]
    }
    catch {
        $message = $_.Exception.GetBaseException().Message 
        Write-Error $message 
        throw "Error parsing ini file: section ${section}"
    }

    $global:SDP_INSTANCE_HOME = -join($global:SDP_GLOBAL_ROOT, "\p4\", $SDPInstance)
    $global:SDP_INSTANCE_BIN_DIR = -join($global:SDP_INSTANCE_HOME, "\bin")
    $global:P4exe =  -join($global:SDP_INSTANCE_BIN_DIR, "\p4.exe")
    $global:P4Dexe =  -join($global:SDP_INSTANCE_BIN_DIR, "\p4d.exe")
    Ensure-PathExists $global:p4exe
    Ensure-PathExists $global:p4dexe
    $global:SDP_INSTANCE_SSL_DIR = -join($global:SDP_INSTANCE_HOME, "\ssl")
    $global:P4CONFIG = "p4config.txt"
    $global:P4ROOT = -join($global:SDP_INSTANCE_HOME, "\root")
    $global:P4LOG = -join($global:SDP_INSTANCE_HOME, "\logs\", $SDPINSTANCE, ".log")
    $global:P4JOURNAL = -join($global:SDP_INSTANCE_HOME, "\logs\journal")
    $global:P4USER = $global:SDP_P4SUPERUSER
    $env:P4TICKETS = -join($global:SDP_INSTANCE_HOME, "\P4Tickets.txt")
    $env:P4TRUST = -join($global:SDP_INSTANCE_HOME, "\P4Trust.txt")
    $env:P4ENVIRO = -join($global:SDP_INSTANCE_HOME, "\P4Enviro.txt")
    $global:SDP_INSTANCE_P4SERVICE_NAME = -join("p4_", $SdpInstance)
    $global:P4PORT = Get-P4PORT
    $global:SCRIPTS_DIR = -join($global:SDP_GLOBAL_ROOT, "\p4\common\bin")
    $global:LOGS_DIR = -join($global:SDP_INSTANCE_HOME, "\logs")
    $global:OFFLINE_DB_DIR = -join($global:SDP_INSTANCE_HOME, "\offline_db")
    $global:CHECKPOINTS_DIR = -join($global:SDP_INSTANCE_HOME, "\checkpoints")
    $global:REMOTE_CHECKPOINTS_DIR = -join($global:REMOTE_DEPOTDATA_ROOT, "\p4\", $global:REMOTE_SDP_INSTANCE, "\checkpoints")
    $global:ADMIN_PASS_FILE = -join($global:SCRIPTS_DIR, "\", $global:ADMIN_PASS_FILENAME)
    $global:EMAIL_PASS_FILE = -join($global:SCRIPTS_DIR, "\", $global:EMAIL_PASS_FILENAME)
    $Global:EDGE_SERVER = $false    # Placeholder for now for comparison with Unix scripts
    # PATH=%SDP_INSTANCE_BIN_DIR%;%SCRIPTS_DIR%;%PATH%
    Ensure-PathExists $global:SDP_INSTANCE_BIN_DIR
    Ensure-PathExists $global:P4ROOT
    Ensure-PathExists $global:LOGS_DIR
    Ensure-PathExists $global:OFFLINE_DB_DIR
    Ensure-PathExists $global:CHECKPOINTS_DIR
    Ensure-PathExists $global:ADMIN_PASS_FILE
}

Function Get-P4PORT {
    $regkey = "HKLM:\SYSTEM\CurrentControlSet\services\${global:SDP_INSTANCE_P4SERVICE_NAME}\parameters"
    (get-itemproperty $regkey).P4PORT
}

Function Get-Date-Time () {
    Get-Date -format "yyyy\/MM\/dd HH\:mm\:ss"
}

Function Ensure-PathExists([string]$path) {
    if (!(Test-Path -path $path)) {
        throw "Error - required path $path doesn't exist!"
    }
}

Function Create-LogFile () {
    if (!$global:LogFileName) {
        throw "ERROR - Global:LogFileName not set"
    }
    $global:LogFile = Join-Path $global:LOGS_DIR $global:LogFileName
    Rotate-CurrentLogFile
    write-debug "Using logfile: $global:logfile"
    write-output "Starting ${global:ScriptTask}" | set-content -path $global:Logfile
}

Function Append-To-File([string]$message, [string]$filename) {
    $OFS = $null
    if (test-path variable:global:OFS) {
        $OFS = $global:OFS
    }
    $global:OFS = "`r`n"
    "$message" | add-content -path $filename
    if ($OFS) {
        $global:OFS = $OFS
    }
}

Function Log([string]$message) {
    # Logs output to file and to console
    $datetime = get-date-time
    write-host "$datetime $message"
    Append-To-File "$datetime $message" $global:Logfile
}

Function LogException([exception]$exception) {
    $datetime = get-date-time
    $message = $exception.message
    write-error -message "$datetime $message" -Exception $exception
    write-output "$datetime $message" | add-content -path $global:Logfile
}

Function Parse-IniFile ($file) {
    $ini = @{}

    # Create a default section if none exist in the file. Like a java prop file.
    $section = "NO_SECTION"
    $ini[$section] = @{}

    switch -regex -file $file {
        "^\[(.+)\]$" {
            $section = $matches[1].Trim().ToLower()
            $ini[$section] = @{}
        }
        "^\s*([^#].+?)\s*=\s*(.*)" {
            $name,$value = $matches[1..2]
            # skip comments that start with semicolon:
            if (!($name.StartsWith(";"))) {
                $ini[$section][$name] = $value.Trim()
            }
        }
    }
    $ini
}

Function Invoke-P4Login () {
    Log "Logging in to P4 Instance"
    $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER login -a < $global:ADMIN_PASS_FILE >> $global:LogFile 2>&1"
    Log $cmd
    cmd /c $cmd
    if ($lastexitcode -ne 0) {
        throw "ERROR - failed to login!"
    }
}

Function Run-ReplicaStatus ([string]$statuslog) {
    Log "Getting replica status"
    $result = & $global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER pull -lj 2>&1
    Log $result
    if ((Test-Path $statuslog -pathtype leaf)) {
        remove-item $statuslog
    }
    Append-To-File "$result" $statuslog
    if ($lastexitcode -ne 0) {
        Log "WARNING - pull command exited with error code"
    }
}

Function Check-OfflineDBExists () {
    Log "Checking offline DB exists"
    $path = Join-Path $global:OFFLINE_DB_DIR "db.counters"
    Log "checking existence of $path"
    if (!(Test-Path $path -pathtype leaf)) {
        throw "ERROR - offline_db doesn't exist!"
    }
}

Function Ensure-CheckpointNotRunning () {
    Log "Ensuring there is no other checkpoint script running"
    $checkFile = Join-Path $global:CHECKPOINTS_DIR "ckp_running.txt"
    Log "checking existence of $checkFile"
    if (Test-Path $checkFile -pathtype leaf) {
        throw "Error - a checkpoint operation is already running!"
    }
    write-output "Checkpoint running" | set-content -path $checkFile
}

Function Signal-CheckpointComplete () {
    $checkFile = Join-Path $global:CHECKPOINTS_DIR "ckp_running.txt"
    remove-item $checkFile
}

Function Get-CurrentJournalCounter () {
    Log "Getting current live journal counter"
    $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER counter journal"
    Log $cmd
    $result = Invoke-Expression $cmd
    Log "Current journal counter is $result"
    if ($lastexitcode -ne 0) {
        throw "Error - failed to get live journal counter!"
    }
    [int]$global:JOURNAL_NUM = $result
    [int]$global:CHECKPOINT_NUM = $global:JOURNAL_NUM + 1        
    Log "Live journal/checkpoint counters ${global:JOURNAL_NUM}/${global:CHECKPOINT_NUM}"
}

Function Get-JournalCounter ([string]$rootdir) {
    $cmd = "$global:P4Dexe -r $rootdir -jd - db.counters"
    Log $cmd
    $rawjournal = Invoke-Expression $cmd | select-string "@journal@" | %{$_.line.split()[4]}
    Log $rawjournal
    $journal = [regex]::match($rawjournal, '^@([0-9]+)@$').Groups[1].Value
    if ($lastexitcode -ne 0) {
        throw "Error - failed to get $rootdir journal counter!"
    }
    return [int]$journal
}

Function Get-OfflineJournalCounter () {
    Log "Getting offline journal counter"
    $journal = Get-JournalCounter $global:OFFLINE_DB_DIR
    Log "Offline journal counter $journal"
    [int]$global:Offline_journal_num = $journal
}

Function Rotate-CurrentLogFile () {
    $Rotate_logname = $global:LOGFILE
    if (Test-Path $Rotate_logname -pathtype leaf) {
        $suffix = (Get-ChildItem $Rotate_logname).lastwritetime | Get-Date -format "yyyyMMdd\-HHmmss"
        $newname = "$Rotate_logname.$suffix"
        write-host "Rotating $Rotate_logname to $newname"
        rename-item -path $Rotate_logname -newname $newname
    }
}

Function Rotate-LogFile () {
    param([string]$Rotate_logname, [bool]$zip)
    if (Test-Path $Rotate_logname -pathtype leaf) {
        $newname = "$Rotate_logname.$global:logid"
        Log "Rotating $Rotate_logname to $newname"
        rename-item -path $Rotate_logname -newname "$newname"
        if ($zip) {
            $gzip = -join($global:SCRIPTS_DIR, "\gzip.exe")
            $cmd = "$gzip $newname"
            Log $cmd
            Invoke-Expression $cmd
        }
    }
}

Function Rotate-Logfiles () {
    Log "Rotating various logfiles"
    $datetime = Get-Date -format "yyyyMMdd\-HHmmss"
    $global:LOGID = "${global:JOURNAL_NUM}.$datetime"

    Rotate-LogFile "${global:sdp_serverid}.log" $true
    Rotate-LogFile "log" $true
    Rotate-LogFile "p4broker.log" $true
    Rotate-LogFile "audit.log" $true
    Rotate-LogFile "sync_replica.log" $false
}

Function Move-Files ([string]$source, [string]$target) {
    Get-ChildItem -Path $source |
        foreach-object {
            $f = $_.FullName
            Log "Moving $f to $target"
            move-item $f $target
        }
}

Function Remove-Files {
    # Remove older files while keeping a specified minimum number
    param([string]$remove_filename, [int]$keepnum)

    $file_path = -join($remove_filename, "*")
    $files = @(Get-ChildItem -Path $file_path | Sort-Object -Property LastWriteTime -Descending)
    if ($files) {
        for ($j = $keepnum; $j -lt $files.count; $j++) {
            $f = $files[$j].FullName
            Log("Removing: $f")
            remove-item $f -Force
        }
    }
}

Function Remove-OldLogs () {
    # Remove old Checkpoint Logs
    # Use KEEPCKPS rather than KEEPLOGS, so we keep the same number
    # of checkpoint logs as we keep checkpoints.
    if ($global:keepckps -eq 0) {
        Log "Skipping cleanup of old checkpoint logs because KEEPCKPS is set to 0."
    } else {
        log "Deleting old checkpoint logs.  Keeping latest $global:KEEPCKPS, per KEEPCKPS setting in sdp_config.ini."
        remove-files $global:P4LOG $global:KEEPCKPS
        remove-files "checkpoint.log" $global:KEEPCKPS
        remove-files "log" $global:KEEPCKPS
        remove-files "p4broker.log" $global:KEEPCKPS
        remove-files "audit.log" $global:KEEPCKPS
        remove-files "sync_replica.log" $global:KEEPCKPS
        remove-files "recreate_offline_db.log" $global:KEEPCKPS
        remove-files "upgrade.log" $global:KEEPCKPS
    }
}

Function Truncate-Journal () {
    Log "Truncating live journal"
    $checkpoint_path = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.", $global:Checkpoint_num, ".gz")
    if (Test-Path $checkpoint_path -pathtype leaf) {
        throw "ERROR - $checkpoint_path already exists, check the backup process"
    }
    if (!$global:EDGE_SERVER) {
        $journalprefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME)
        $journalpath = -join($journalprefix, ".jnl.", $global:JOURNAL_NUM)
        if (Test-Path $journalpath -pathtype leaf) {
            throw "ERROR - $journalpath already exists, check the backup process"
        }
        log "Truncating journal..."
        # 'p4d -jj' does a copy-then-delete, instead of a simple mv.
        # during 'p4d -jj' the perforce server will hang the responses to clients.
        $cmd = "$global:P4Dexe -r $global:P4ROOT -J $global:P4JOURNAL -jj $journalprefix"
        Log $cmd
        $result = Invoke-Expression $cmd
        Log $result
        if ($lastexitcode -ne 0) {
            throw "ERROR - attempting to truncate journal"
        }
    }
}

Function Run-Checkpoint () {
    $checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME)
    $cmd = "$global:P4DEXE -r $global:P4ROOT -J $global:P4JOURNAL -jc -Z $checkpoint_prefix"
    Log $cmd
    $result = Invoke-Expression $cmd
    Log $result
    if ($lastexitcode -ne 0) {
        throw "ERROR - attempting to checkpoint live database"
    }
}

Function Replay-JournalsToOfflineDB () {
    Log "Applying all oustanding journal files to offline_db"
    for ($j = $global:Offline_journal_num; $j -le $global:JOURNAL_NUM; $j++) {
        $journalprefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME)
        $journalpath = -join($journalprefix, ".jnl.", $j)
        $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -jr $journalpath"
        Log $cmd
        $result = Invoke-Expression $cmd
        Log $result
        if ($lastexitcode -ne 0) {
            throw "ERROR - attempting to replay journal"
        }
    }
}

function Create-OfflineCheckpoint {
    Log "Creating offline checkpoint"
    $checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.")
    if ($global:limit_one_daily_checkpoint -ne 0) {
        $curr_date = Get-Date -format "yyyyMMdd"
        $checkpoint_path = -join($checkpoint_prefix, "*.gz")
        $files = @(Get-ChildItem -Path $checkpoint_path | Sort-Object -Property LastWriteTime -Descending)
        if (!$files -or $files.count -eq 0) {
           Log "LIMIT_ONE_DAILY_CHECKPOINT is set to true, but no checkpoint exists."
        } else {
            $ckp_date = $files[0].LastWriteTime | Get-Date -format "yyyyMMdd"
            if ($curr_date -eq $ckp_date) {
                Log "A checkpoint was already created today, and LIMIT_ONE_DAILY_CHECKPOINT is set to true."
                Log "Skipping offline checkpoint dump."
                return
            }
        }
    }
    $checkpoint_path = -join($checkpoint_prefix, $global:checkpoint_num, ".gz")
    $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -jd -z $checkpoint_path"
    Log $cmd
    $result = Invoke-Expression $cmd
    Log $result
    if ($lastexitcode -ne 0) {
        throw "ERROR - attempting to create offline checkpoint"
    }
}

Function Check-OfflineDBUsable () {
    $offline_db_usable = -join($global:OFFLINE_DB_DIR, "\offline_db_usable.txt")
    if (!(test-path $offline_db_usable -pathtype leaf)) {
        $msg = "Offline database not in a usable state. Check the backup process."
        Log $msg
        throw $msg
    }
}

Function Recreate-OfflineDBFiles () {
    Log "Recreating offline db files"
    $offline_db_usable = -join($global:OFFLINE_DB_DIR, "\offline_db_usable.txt")
    $checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.")
    $checkpoint_path = -join($checkpoint_prefix, "*.gz")
    $files = @(Get-ChildItem -Path $checkpoint_path | Sort-Object -Property LastWriteTime -Descending)
    if (!$files -or $files.count -eq 0) {
        $msg = "No checkpoints found - run live-checkpoint.ps1"
        Log $msg
        throw $msg
    }
    if (test-path $offline_db_usable -pathtype leaf) {
        remove-item $offline_db_usable
    }
    remove-files "${global:OFFLINE_DB_DIR}\db." 0
    
    Log "Recovering from latest checkpoint found"
    $checkpoint_path = $files[0]
    $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -jr -z $checkpoint_path"
    Log $cmd
    $result = Invoke-Expression $cmd
    Log $result
    if ($lastexitcode -ne 0) {
        throw "ERROR - attempting to recover offline db files"
    }    
    "Offline db file restored successfully." | set-content -path $offline_db_usable
}

Function Remove-OldCheckpointsAndJournals () {
    if ($global:keepckps -eq 0) {
        Log "Skipping cleanup of old checkpoints because KEEPCKPS is set to 0."
    } else {
        log "Deleting obsolete checkpoints and journals.  Keeping latest $global:KEEPCKPS, per KEEPCKPS setting in sdp_config.ini."
        $checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.")
        $journal_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".jnl.")
        # We multiply KEEPCKP by 2 for the ckp files because of the md5 files.
        remove-files $checkpoint_prefix ($global:KEEPCKPS * 2)
        remove-files $journal_prefix ($global:KEEPCKPS)
    }
}

Function run-cmd ([string]$cmd) {
    Log $cmd
    $result = Invoke-Expression $cmd
    Log $result
}

Function Log-DiskSpace () {
    log "Checking disk space..."
    $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER diskspace"
    run-cmd "$cmd"
}

Function set-counter () {
    Invoke-P4Login
    $datetime = get-date-time
    $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER counter lastSDPCheckpoint ""$datetime"""
    run-cmd -cmd $cmd
}

Function Compare-JournalNumbers () {
    # Ensures offline and live db counters are the same - avoids using out-of-date offline_db
    $offlineJournal = Get-JournalCounter $global:OFFLINE_DB_DIR
    $liveJournal = Get-JournalCounter $global:P4ROOT
    if ($liveJournal -ne $offlineJournal) {
      log "$global:P4ROOT journal number is: $liveJournal"
      log "$global:OFFLINE_DB_DIR journal number is: $offlineJournal"
      throw "$global:P4ROOT and $global:OFFLINE_DB_DIR journal numbers do not match." 
    }
}

Function Move-OfflineDBToLive () {
    # Compare the Offline and Master journal numbers before switching to make sure they match.
    Compare-JournalNumbers
    log "Switching out db files..."
    $savedir = "${global:P4ROOT}\save"
    if (!(test-path -path $savedir -type container)) {
        New-Item $savedir -ItemType directory
    }
    remove-files "$savedir\db." 0
    move-files "${global:P4ROOT}\db.*" $savedir
    move-files "${global:offline_db_dir}\db.*" $global:P4ROOT
    remove-item "${global:offline_db_dir}\offline_db_usable.txt"
}

Function Test-IsAdmin {
    ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")
}

Function Check-AdminPrivileges {
    if (!(Test-IsAdmin)) {
        throw "Please run this script with admin priviliges"
    }
}

Function Stop-LiveService () {
    Stop-Service $global:SDP_INSTANCE_P4SERVICE_NAME
}

Function Start-LiveService () {
    Start-Service $global:SDP_INSTANCE_P4SERVICE_NAME
}

Function send-email ([string]$subject, [string]$logfilename) {
    Log "Sending email with subject: $subject"
    if ($logfilename) {
        $contents = get-content $logfilename
    } else {
        $contents = get-content $global:logfile
    }
    $SMTPServer = $global:mailhost
    $SMTPPort = "25"
    if ($global:mailhostport -ne "") {
        $SMTPPort = $global:mailhostport
    }
    try {
        $email_password = ""
        if ((test-path $global:email_pass_file -pathtype leaf)) {
            Log "Found  $global:email_pass_file"
            $email_password = get-content $global:email_pass_file
        }
        $message = New-Object System.Net.Mail.MailMessage
        $message.subject = $subject
        $message.body = $contents -join "<br>`n"
        $message.isbodyhtml = $true
        $message.to.add($global:maillist)
        $message.from = $global:mailfrom
        $smtp = New-Object System.Net.Mail.SmtpClient($SMTPServer, $SMTPPort)
        if ($SMTPPort -ne "25") {
            $smtp.EnableSSL = $true
        }
        if ($email_password -ne "") {
            $smtp.Credentials = New-Object System.Net.NetworkCredential($global:mailfrom, $email_password)
        }
        $smtp.send($message)
    }
    Catch
    {
        $ErrorMessage = $_.Exception.Message
        Log "Failed to send email to ${global:mailhost}: ${ErrorMessage}"
    }
}