SDP-functions.ps1 #3

# ============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
# ----------------------------------------------------------------------------

<#
    .Synopsis
        Various SDP related shared functions
        
    .Description
        Functions intended to be called by other scripts.
        
#>

Function Parse-SDP-Config-File([string]$scriptpath) {
    $SDPConfigFile = Split-Path -parent $scriptpath | Join-Path -childpath "..\..\config\sdp_config.ini"
    $SDPConfigFile = Resolve-Path $SDPConfigFile
    $ini = Parse-Inifile($SDPConfigFile)

    $section = "${SDPInstance}:${env:computername}".tolower()
    if ($ini[$section] -eq $null) {
        throw("Section $section not found in ${SDPConfigFile}")
    }
    
    try {
        $global:sdp_global_root = $ini[$section]["sdp_global_root"]
        $global:sdp_serverid = $ini[$section]["sdp_serverid"]
        $global:sdp_p4serviceuser = $ini[$section]["sdp_p4serviceuser"]
        $global:sdp_p4superuser = $ini[$section]["sdp_p4superuser"]
        $global:admin_pass_filename = $ini[$section]["admin_pass_filename"]
        $global:mailfrom = $ini[$section]["mailfrom"]
        $global:maillist = $ini[$section]["maillist"]
        $global:mailhost = $ini[$section]["mailhost"]
        $global:python = $ini[$section]["python"]
        $global:remote_depotdata_root = $ini[$section]["remote_depotdata_root"]
        $result = [int32]::TryParse($ini[$section]["keepckps"], [ref]$global:keepckps)
        $result = [int32]::TryParse($ini[$section]["keeplogs"], [ref]$global:keeplogs)
        $global:limit_one_daily_checkpoint = $ini[$section]["limit_one_daily_checkpoint"]
        $global:remote_sdp_instance = $ini[$section]["remote_sdp_instance"]
        $global:p4target = $ini[$section]["p4target"]
    }
    catch {
        $message = $_.Exception.GetBaseException().Message 
        Write-Error $message 
        throw("Error parsing ini file: section ${section}")
    }

    $global:SDP_INSTANCE_HOME = -join($global:SDP_GLOBAL_ROOT, "\p4\", $SDPInstance)
    $global:SDP_INSTANCE_BIN_DIR = -join($global:SDP_INSTANCE_HOME, "\bin")
    $global:P4exe =  -join($global:SDP_INSTANCE_BIN_DIR, "\p4.exe")
    $global:P4Dexe =  -join($global:SDP_INSTANCE_BIN_DIR, "\p4d.exe")
    Ensure-Path-Exists $global:p4exe
    Ensure-Path-Exists $global:p4dexe
    $global:SDP_INSTANCE_SSL_DIR = -join($global:SDP_INSTANCE_HOME, "\ssl")
    $global:P4CONFIG = "p4config.txt"
    $global:P4ROOT = -join($global:SDP_INSTANCE_HOME, "\root")
    $global:P4LOG = -join($global:SDP_INSTANCE_HOME, "\logs\", $SDPINSTANCE, ".log")
    $global:P4JOURNAL = -join($global:SDP_INSTANCE_HOME, "\logs\journal")
    $global:P4USER = $global:SDP_P4SUPERUSER
    $global:P4TICKETS = -join($global:SDP_INSTANCE_BIN_DIR, "\P4Tickets.txt")
    $global:P4TRUST = -join($global:SDP_INSTANCE_BIN_DIR, "\P4Trust.txt")
    $global:P4ENVIRO = -join($global:SDP_INSTANCE_BIN_DIR, "\P4Enviro.txt")
    $global:SDP_INSTANCE_P4SERVICE_NAME = -join("P4_", $SdpInstance)
    $global:P4PORT = Get-P4PORT
    $global:SCRIPTS_DIR = -join($global:SDP_GLOBAL_ROOT, "\p4\common\bin")
    $global:LOGS_DIR = -join($global:SDP_INSTANCE_HOME, "\logs")
    $global:OFFLINE_DB_DIR = -join($global:SDP_INSTANCE_HOME, "\offline_db")
    $global:CHECKPOINTS_DIR = -join($global:SDP_INSTANCE_HOME, "\checkpoints")
    $global:REMOTE_CHECKPOINTS_DIR = -join($global:REMOTE_DEPOTDATA_ROOT, "\p4\", $global:REMOTE_SDP_INSTANCE, "\checkpoints")
    $global:ADMIN_PASS_FILE = -join($global:SCRIPTS_DIR, "\", $global:ADMIN_PASS_FILENAME)
    $Global:EDGE_SERVER = $false    # Placeholder for now for comparison with Unix scripts
    # PATH=%SDP_INSTANCE_BIN_DIR%;%SCRIPTS_DIR%;%PATH%
    Ensure-Path-Exists $global:SDP_INSTANCE_BIN_DIR
    Ensure-Path-Exists $global:P4ROOT
    Ensure-Path-Exists $global:LOGS_DIR
    Ensure-Path-Exists $global:OFFLINE_DB_DIR
    Ensure-Path-Exists $global:CHECKPOINTS_DIR
    Ensure-Path-Exists $global:ADMIN_PASS_FILE
}

Function Get-P4PORT {
    $regkey = "HKLM:\SYSTEM\CurrentControlSet\services\${global:SDP_INSTANCE_P4SERVICE_NAME}\parameters"
    (get-itemproperty $regkey).P4PORT
}

Function Get-Date-Time () {
    Get-Date -format "yyyy\/MM\/dd HH\:mm\:ss"
}

Function Ensure-Path-Exists([string]$path) {
    if (!(Test-Path -path $path)) {
        throw("Error - required path $path doesn't exist!")
    }
}

Function Create-Log-File () {
    if (!$global:LogFileName) {
        throw("ERROR - Global:LogFileName not set")
    }
    $global:LogFile = Join-Path $global:LOGS_DIR $global:LogFileName
    write-debug "Using logfile: $global:logfile"
    write-output "" | set-content -path $global:Logfile
}

Function Log($message) {
    # Logs output to file and to console
    $datetime = get-date-time
    write-output "$datetime $message"
    write-output "$datetime $message" | add-content -path $global:Logfile
}

Function Parse-IniFile ($file) {
    $ini = @{}

    # Create a default section if none exist in the file. Like a java prop file.
    $section = "NO_SECTION"
    $ini[$section] = @{}

    switch -regex -file $file {
        "^\[(.+)\]$" {
            $section = $matches[1].Trim().ToLower()
            $ini[$section] = @{}
        }
        "^\s*([^#].+?)\s*=\s*(.*)" {
            $name,$value = $matches[1..2]
            # skip comments that start with semicolon:
            if (!($name.StartsWith(";"))) {
                $ini[$section][$name] = $value.Trim()
            }
        }
    }
    $ini
}

Function Invoke-P4Login () {
    Log("Logging in to P4 Instance")
    $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER login -a < $global:ADMIN_PASS_FILE"
    Log($cmd)
    cmd /c $cmd
    if (($result -eq 0) -or ($lastexitcode -ne 0)) {
        throw("ERROR - failed to login!")
    }
}

Function Check-Offline-DB-Exists () {
    Log("Checking offline DB exists")
    $path = Join-Path $global:OFFLINE_DB_DIR "db.counters"
    Log("checking existence of $path")
    if ( -Not (Test-Path $path -pathtype leaf)) {
        throw("ERROR - offline_db doesn't exist!")
    }
}

Function Ensure-Checkpoint-Not-Running () {
    Log("Ensuring there is no other checkpoint script running")
    $checkFile = Join-Path $global:CHECKPOINTS_DIR "ckp_running.txt"
    Log("checking existence of $checkFile")
    if (Test-Path $checkFile -pathtype leaf) {
        throw("Error - a checkpoint operation is already running!")
    }
    write-output "Checkpoint running" | set-content -path $checkFile
}

Function Signal-Checkpoint-Complete () {
    $checkFile = Join-Path $global:CHECKPOINTS_DIR "ckp_running.txt"
    remove-item $checkFile
}


Function Get-Current-Journal-Counter () {
    Log("Getting current live journal counter")
    $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER counter journal"
    Log($cmd)
    $result = Invoke-Expression $cmd
    Log("Current journal counter is $result")
    if (($result -eq 0) -or ($lastexitcode -ne 0)) {
        throw("Error - failed to get live journal counter!")
    }
    [int]$global:JOURNAL_NUM = $result
    [int]$global:CHECKPOINT_NUM = $global:JOURNAL_NUM + 1        
    Log("Live journal/checkpoint counters ${global:JOURNAL_NUM}/${global:CHECKPOINT_NUM}")
}

Function Get-Offline-Journal-Counter () {
    Log("Getting offline journal counter")
    $cmd = "$global:P4Dexe -r $global:OFFLINE_DB_DIR -jd - db.counters"
    Log($cmd)
    $rawjournal = Invoke-Expression $cmd | select-string "@journal@" | %{$_.line.split()[4]}
    $journal = [regex]::match($rawjournal, '^@([0-9]+)@$').Groups[1].Value
    if (($result -eq 0) -or ($lastexitcode -ne 0)) {
        throw("Error - failed to get offline journal counter!")
    }
    Log("Offline journal counter $journal")
    [int]$global:Offline_journal_num = $journal
}

Function Rotate-log-file () {
    param([string]$Rotate_logname, [bool]$zip)
    Log("Rotate-log-file $Rotate_logname, $zip")
    if (Test-Path $Rotate_logname -pathtype leaf) {
        $newname = "$Rotate_logname.$global:logid"
        Log("Rotating $Rotate_logname to $newname")
        rename-item -path $Rotate_logname -newname "$newname"
        if ($zip) {
            $gzip = -join($global:SCRIPTS_DIR, "\gzip.exe")
            $cmd = "$gzip $newname"
            Log($cmd)
            Invoke-Expression $cmd
        }
    }
}

Function Rotate-Last-Run-Logs () {
    Log("Rotating last run logs")
    $datetime = Get-Date -format "yyyyMMdd\-HHmmss"
    $global:LOGID = "${global:JOURNAL_NUM}.$datetime"

    rotate-log-file $global:LOGFILE $false
    rotate-log-file "${global:sdp_serverid}.log" $true
    rotate-log-file "log" $true
    rotate-log-file "p4broker.log" $true
    rotate-log-file "audit.log" $true
    rotate-log-file "sync_replica.log" $false
}

Function remove-files {
    param([string]$remove_filename, [int]$keepnum)

    $file_path = -join($remove_filename, "*")
    $files = Get-ChildItem -Path $file_path | Sort-Object -Property LastWriteTime -Descending
    for ($j = $keepnum; $j -lt $files.count; $j++) {
        $f = $files[$j].FullName
        Log("Removing: $f")
        remove-item $f -Force
    }
}

Function remove-old-logs () {
    # Remove old Checkpoint Logs
    # Use KEEPCKPS rather than KEEPLOGS, so we keep the same number
    # of checkpoint logs as we keep checkpoints.
    if ($global:keepckps -eq 0) {
        Log("Skipping cleanup of old checkpoint logs because KEEPCKPS is set to 0.")
    } else {
        log("Deleting old checkpoint logs.  Keeping latest $global:KEEPCKPS, per KEEPCKPS setting in sdp_config.ini.")
        remove-files "checkpoint.log" $global:KEEPCKPS
        remove-files "log" $global:KEEPCKPS
        remove-files "p4broker.log" $global:KEEPCKPS
        remove-files "audit.log" $global:KEEPCKPS
        remove-files "sync_replica.log" $global:KEEPCKPS
        remove-files "recreate_offline_db.log" $global:KEEPCKPS
        remove-files "upgrade.log" $global:KEEPCKPS
    }
}

Function truncate-journal () {
    Log("Truncating live journal")
    $checkpoint_path = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.", $global:Checkpoint_num, ".gz")
    if (Test-Path $checkpoint_path -pathtype leaf) {
        throw("ERROR - $checkpoint_path already exists, check the backup process")
    }
    if (!$global:EDGE_SERVER) {
        $journalprefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME)
        $journalpath = -join($journalprefix, ".jnl.", $global:JOURNAL_NUM)
        if (Test-Path $journalpath -pathtype leaf) {
            throw("ERROR - $journalpath already exists, check the backup process")
        }
        log("Truncating journal...")
        # 'p4d -jj' does a copy-then-delete, instead of a simple mv.
        # during 'p4d -jj' the perforce server will hang the responses to clients.
        $cmd = "$global:P4Dexe -r $global:P4ROOT -J $global:P4JOURNAL -jj $journalprefix"
        Log($cmd)
        $result = Invoke-Expression $cmd
        Log($result)
        if ($lastexitcode -ne 0) {
            throw("ERROR - attempting to truncate journal")
        }
    }
}

Function replay-journals-to-offline-db () {
    Log("Applying all oustanding journal files to offline_db")
    for ($j = $global:Offline_journal_num; $j -le $global:JOURNAL_NUM; $j++) {
        $journalprefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME)
        $journalpath = -join($journalprefix, ".jnl.", $j)
        $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -jr $journalpath"
        Log($cmd)
        $result = Invoke-Expression $cmd
        Log($result)
        if ($lastexitcode -ne 0) {
            throw("ERROR - attempting to replay journal")
        }
    }
}

function create-offline-checkpoint {
    Log("Creating offline checkpoint")
    $checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.")
    if ($global:limit_one_daily_checkpoint -ne 0) {
        $curr_date = Get-Date -format "yyyyMMdd"
        $checkpoint_path = -join($checkpoint_prefix, "*.gz")
        $files = Get-ChildItem -Path $checkpoint_path | Sort-Object -Property LastWriteTime -Descending
        if ($files.length -eq 0) {
           Log("LIMIT_ONE_DAILY_CHECKPOINT is set to true, but no checkpoint exists.")
        } else {
            $ckp_date = $files[0].LastWriteTime | Get-Date -format "yyyyMMdd"
            if ($curr_date -eq $ckp_date) {
                Log("A checkpoint was already created today, and LIMIT_ONE_DAILY_CHECKPOINT is set to true.")
                Log("Skipping offline checkpoint dump.")
                return
            }
        }
    }
    $checkpoint_path = -join($checkpoint_prefix, $global:checkpoint_num, ".gz")
    $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -jd -z $checkpoint_path"
    Log($cmd)
    $result = Invoke-Expression $cmd
    Log($result)
    if ($lastexitcode -ne 0) {
        throw("ERROR - attempting to create offline checkpoint")
    }
}

Function recreate-offline-db-files () {
    Log("Recreating offline db files")
    $offline_db_usable = -join($global:OFFLINE_DB_DIR, "\offline_db_usable.txt")
    $checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.")
    $checkpoint_path = -join($checkpoint_prefix, "*.gz")
    $files = Get-ChildItem -Path $checkpoint_path | Sort-Object -Property LastWriteTime -Descending
    if ($files.length -eq 0) {
        $msg = "No checkpoints found - run live_checkpoint.sh"
        Log($msg)
        throw($msg)
    }
    if (test-path $offline_db_usable -pathtype leaf) {
        remove-item $offline_db_usable
    }
    $db_files = join-path $global:OFFLINE_DB_DIR "*.db"
    remove-item $db_files
    
    Log("Recovering from latest checkpoint found")
    $checkpoint_path = $files[0]
    $cmd = "$global:P4DEXE -r $global:OFFLINE_DB_DIR -jr -z $checkpoint_path"
    Log($cmd)
    $result = Invoke-Expression $cmd
    Log($result)
    if ($lastexitcode -ne 0) {
        throw("ERROR - attempting to recover offline db files")
    }    
    "Offline db file restored successfully." | set-content -path $offline_db_usable
}

Function remove-old-checkpoints-and-journals () {
    if ($global:keepckps -eq 0) {
        Log("Skipping cleanup of old checkpoints because KEEPCKPS is set to 0.")
    } else {
        log("Deleting obsolete checkpoints and journals.  Keeping latest $global:KEEPCKPS, per KEEPCKPS setting in sdp_config.ini.")
        $checkpoint_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".ckp.")
        $journal_prefix = -join($global:CHECKPOINTS_DIR, "\", $global:SDP_INSTANCE_P4SERVICE_NAME, ".jnl.")
        # We multiply KEEPCKP by 2 for the ckp files because of the md5 files.
        remove-files $checkpoint_prefix ($global:KEEPCKPS * 2)
        remove-files $journal_prefix ($global:KEEPCKPS)
    }
}

Function run-cmd ([string]$cmd) {
    Log($cmd)
    $result = Invoke-Expression $cmd
    Log($result)
}

Function check-disk-space () {
    log("Checking disk space...")
    $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER diskspace"
    run-cmd "$cmd"
}

Function set-counter () {
    Invoke-P4Login
    $datetime = get-date-time
    $cmd = "$global:P4exe -p $global:P4PORT -u $global:SDP_P4SUPERUSER counter lastSDPCheckpoint ""$datetime"""
    run-cmd -cmd $cmd
}

Function send-email ([string]$subject) {
    Log("Sending email with subject: $subject")
    $contents = get-content $global:logfile
    $SMTPServer = $global:mailhost
    $SMTPPort = "25"
    try {
        $message = New-Object System.Net.Mail.MailMessage
        $message.subject = $subject
        $message.body = $contents
        $message.to.add($global:maillist)
        $message.from = $global:mailfrom
        $smtp = New-Object System.Net.Mail.SmtpClient($SMTPServer, $SMTPPort);
        #$smtp.EnableSSL = $true
        #$smtp.Credentials = New-Object System.Net.NetworkCredential($Username, $Password);
        $smtp.send($message)
    }
    Catch
    {
        $ErrorMessage = $_.Exception.Message
        Log("Failed to send email to ${global:mailhost}: ${ErrorMessage}")
    }
}