#!/usr/bin/env python
import os
import pwd
import grp
import sys
import subprocess
import pdb
from P4 import P4, P4Exception
sys.path.append("/usr/local/lib/netapp-manageability-sdk-9.3/lib/python/NetApp")
from NaServer import *
import ssl
try:
_create_unverified_https_context = ssl._create_unverified_context
except AttributeError:
# Legacy Python that doesn't verify HTTPS certificates by default
pass
else:
# Handle target environment that doesn't support HTTPS verification
ssl._create_default_https_context = _create_unverified_https_context
if sys.version_info[0] >= 3:
from configparser import ConfigParser
else:
from ConfigParser import ConfigParser
# ---------------------------------------------------------------------------
# Singleton class (of sorts) for loading configuration
# ---------------------------------------------------------------------------
class Config:
def __init__(self, file):
self.config = ConfigParser()
try:
self.config.read(file)
except:
print("Could not read config file: %s" % file)
sys.exit(2)
def get(self, key):
parts = key.split(".")
try:
value = self.config.get(parts[0], parts[1])
except:
print("Could not access config: %s" % key)
sys.exit(2)
return value
dir = os.path.dirname(os.path.realpath(__file__))
config = Config(dir + "/flex.cfg")
FLEX_SNAP = config.get("p4.snap")
FLEX_CLONE = config.get("p4.clone")
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# NetApp Server connection as 'admin' user
# ---------------------------------------------------------------------------
class NaFlex:
def __init__(self):
self.server = config.get("NaServer.server")
self.port = config.get("NaServer.port")
self.user = config.get("NaServer.admin_user")
self.passwd = config.get("NaServer.admin_passwd")
self.vserver = config.get("NaServer.vserver")
self.transport = config.get("NaServer.transport")
self.style = config.get("NaServer.style")
self.server_type = config.get("NaServer.server_type")
self.aggr = config.get("NaServer.aggr")
self.mount = config.get("NaServer.mount_base")
# Sets the filer properties for connection
# @returns the NaServer connection
def get(self):
s = NaServer(self.server, 1 , 15)
s.set_server_type(self.server_type)
resp = s.set_style(self.style)
if (resp and resp.results_errno() != 0) :
r = resp.results_reason()
print ("Failed to set authentication style " + r + "\n")
sys.exit (2)
s.set_admin_user(self.user, self.passwd)
resp = s.set_transport_type(self.transport)
if (resp and resp.results_errno() != 0) :
r = resp.results_reason()
print ("Unable to set HTTP transport " + r + "\n")
sys.exit (2)
s.set_vserver(self.vserver)
s.set_port(self.port)
return s
# Creates the volume based on name and size
def create(self, name, size, junction_path, uid, gid):
ontapversion = self.getontapversion()
api = NaElement("volume-create")
api.child_add_string("containing-aggr-name", self.aggr)
api.child_add_string("size", size)
api.child_add_string("volume", name)
# api.child_add_string("junction-active", "true")
api.child_add_string("junction-path", junction_path)
if "9." in ontapversion:
#ru the ontap 9 commands for chowning
api.child_add_string("group-id", gid)
api.child_add_string("user-id", uid)
xo = self.get().invoke_elem(api)
if (xo.results_status() == "failed"):
raise NAException(xo.sprintf())
def snapshot_delete(self, volname, snapname):
api = NaElement("snapshot-delete")
api.child_add_string("volume", volname)
api.child_add_string("snapshot", snapname)
#api.child_add_string("vserver", "P4FlexServer")
# initialize exit message
exit_msg = ""
# invoke snapshot delete
# print ("\nvolume name is %s" %volname)
# print ("\nsnapshot name is %s" %snapname)
xo = self.get().invoke_elem(api)
if (xo.results_status() == "failed"):
exit_msg += "ERROR failed to delete snapshot"
exit_msg += " Reason:" + xo.results_reason()
# return exit message - no message if successful
return exit_msg
# Takes a snapshot of specified volume
def snapshot(self, volname, snapname):
api = NaElement("snapshot-create")
api.child_add_string("volume", volname)
api.child_add_string("snapshot", snapname)
xo = self.get().invoke_elem(api)
if (xo.results_status() == "failed"):
raise NAException(xo.sprintf())
# Creates a volume clone based on the snapshot of parent volume
def clone(self, cvolname, psnapname, pvolname, uid, gid,junction_path):
api = NaElement("volume-clone-create")
api.child_add_string("junction-active", "true")
jpath = junction_path
api.child_add_string("junction-path", jpath)
api.child_add_string("parent-snapshot", psnapname)
api.child_add_string("parent-volume", pvolname)
api.child_add_string("volume", cvolname)
# force the flexclone to use thin provisioning even if the parent
# volume is thick previsioned. without this, the clone will not have
# any space savings. it will take the same reserved space as the parent.
api.child_add_string("space-reserve", "none")
ontapversion = self.getontapversion()
# print ("\nhi")
# print ("ontap version is %s " %ontapversion)
if "9." in ontapversion:
#run the ontap 9 commands for chowning
api.child_add_string("gid", gid)
api.child_add_string("uid", uid)
xo = self.get().invoke_elem(api)
if (xo.results_status() == "failed"):
raise NAException(xo.sprintf())
# def modvolumeown:
#A function attempting to modify volume ownership
# api = NaElement("volume-clone-create")
# api.child_add_string("junction-active", "true")
# jpath = "/" + cvolname
def ckclone(self, vname):
# Build the input values for the api query call
#This function checks if a volume has clones - True= has clones
api = NaElement("volume-clone-get-iter")
# now ck if api has vname as master volume
xo = self.get().invoke_elem(api)
if(xo.results_status() == "failed"):
raise NAException(xo.sprintf())
vollist = xo.child_get("attributes-list")
if (vollist == None):
return "False"
for vol_data in vollist.children_get():
clone_name = vol_data.child_get_string( "volume");
parent_volname = vol_data.child_get_string( "parent-volume" );
if parent_volname==vname:
return "True"
return "False"
def getontapversion(self):
api = NaElement ("system-get-version")
xo = self.get().invoke_elem(api)
if(xo.results_status() == "failed"):
raise NAException(xo.sprintf())
sysversion=xo.child_get_string("version")
return sysversion
# Deletes the volume by unmounting, offlining and then deleting volume from vserver
def delete(self, volname):
# Check to make sure we own the volume to delete
# st = os.stat(self.mount + "/" + volname)
# user = self.call.getUser()
# uid = pwd.getpwname(user).pw_uid
# gid = pwd.getpwname(user).pw_gid
# if (uid != st.st_uid):
# raise NAException("You do not have permission to delete volume. Owner of volume is %s." % (pwd.getpwuid(st.st_uid).pw_name))
# Check to make sure volume passed in is not vserver root or node volume
# Root and node volumes are required to keep info on state of vserver and node
api = NaElement("volume-get-iter")
# Build the input values for the api query call
xi1 = NaElement("query")
api.child_add(xi1)
xi2 = NaElement("volume-attributes")
xi1.child_add(xi2)
xi3 = NaElement("volume-id-attributes")
xi2.child_add(xi3)
xi3.child_add_string("name", volname)
xo = self.get().invoke_elem(api)
if (xo.results_status() == "failed"):
raise NAException(xo.sprintf())
vlist = xo.child_get("attributes-list")
vsattrs = None
if (vlist != None):
vol = vlist.child_get("volume-attributes")
vsattrs = vol.child_get("volume-state-attributes")
if (vsattrs != None):
isvroot = vsattrs.child_get_string("is-vserver-root")
isnroot = vsattrs.child_get_string("is-node-root")
if ((isvroot == "true") or (isnroot == "true")):
raise NAException("Not authorized to delete vserver root-volume %s. Go directly to the NetApp Filer to conduct this operation" % (volname))
# Unmount
api = NaElement("volume-unmount")
api.child_add_string("force", "false")
api.child_add_string("volume-name", volname)
xo = self.get().invoke_elem(api)
if (xo.results_status() == "failed"):
raise NAException(xo.sprintf())
# Offline
api = NaElement("volume-offline")
api.child_add_string("name", volname)
xo = self.get().invoke_elem(api)
if ((xo.results_status() == "failed") and (xo.results_errno() != "13042")):
raise NAException(xo.sprintf())
# Delete
api = NaElement("volume-destroy")
api.child_add_string("name", volname)
xo = self.get().invoke_elem(api)
if (xo.results_status() == "failed"):
raise NAException(xo.sprintf())
# Prints out the list of volumes
def vlist (self):
list = []
# Get First Set of mrecs (100) records
mrecs = "100"
api = NaElement("volume-get-iter")
api.child_add_string("max-records", mrecs)
xo = self.get().invoke_elem(api)
if(xo.results_status() == "failed"):
raise NAException(xo.sprintf())
# Set up to get next set of records
tag = xo.child_get_string("next-tag")
# Add vollist results to list and then Get Following set of records
# Break from loop when the vollist is None or next-tag is None
while True:
vollist = xo.child_get("attributes-list")
if (vollist == None):
break
for vol in vollist.children_get():
volattrs = vol.child_get("volume-id-attributes")
volpname = ""
volstateattrs = vol.child_get("volume-state-attributes")
if (volstateattrs != None):
state = volstateattrs.child_get_string("state")
isroot = volstateattrs.child_get_string("is-vserver-root")
volcattrs = vol.child_get("volume-clone-attributes")
if (volcattrs != None):
volpattrs = volcattrs.child_get("volume-returnclone-parent-attributes")
if (volpattrs != None):
volpname = volpattrs.child_get_string("name")
# Only print out volume if volume is online, not a clone, and not vserver root volume
if (state == "online") and (volpname == "") and (isroot != "true"):
list.append(volattrs);
api = NaElement("volume-get-iter")
api.child_add_string("max-records", mrecs)
api.child_add_string("tag", tag)
xo = self.get().invoke_elem(api)
if(xo.results_status() == "failed"):
raise NAException(xo.sprintf())
# Get next tag which indicates if there are more records
tag = xo.child_get_string("next-tag")
if (tag == None):
break
return list
# Print out list of snapshots based on volume name
def slist(self, vname):
# Get First Set of records
api = NaElement("snapshot-list-info")
api.child_add_string("volume", vname)
xo = self.get().invoke_elem(api)
if (xo.results_errno() != 0) :
raise NAException(xo.sprintf())
# # get snapshot list
snapshotlist = xo.child_get("snapshots")
if ((snapshotlist != None) and (snapshotlist != "")) :
# iterate through snapshot list
snapshots = snapshotlist.children_get()
for ss in snapshots:
sname = ss.child_get_string("name")
print("snapshot:%s" % (sname))
else:
print("No snapshots for volume " + vname)
# Print out volume name and their snapshots
def slistall(self):
# Get first set of records defined by mrecs
mrecs = "100"
api = NaElement("snapshot-get-iter")
api.child_add_string("max-records", mrecs)
xo = self.get().invoke_elem(api)
if(xo.results_status() == "failed"):
raise NAException(xo.sprintf())
# Set up tag to get next set of records
tag = xo.child_get_string("next-tag")
# Need to go thru this loop at least once before checking for next tag
# Break out of loop if snaplist is None or next-tag is None
while True:
# Get list of snapshot-info attributes
snaplist = dict()
snaplist = xo.child_get("attributes-list")
if (snaplist == None):
break
# Go thru list and print out volume and snapshot name
for snap in snaplist.children_get():
print("snapshot: %s:%s" % (snap.child_get_string("volume"), snap.child_get_string("name")))
# Get next set of records
api = NaElement("snapshot-get-iter")
api.child_add_string("max-records", mrecs)
xo = self.get().invoke_elem(api)
if(xo.results_status() == "failed"):
raise NAException(xo.sprintf())
# Tag indicates if there are more records
tag = xo.child_get_string("next-tag")
# Break out of loop. Tag of None indicates no more records
if (tag == None):
break
def list_flexclones(self, clonelist):
footprintdict =dict()
mrecs=100
# Get the exact method from volume-footprint-get-iter
# get the footprints of all volumes first
bpi = NaElement("volume-footprint-get-iter")
# bpi.child_add_string("footprint-info", "total-footprint")
bpi.child_add_string("max-records", mrecs)
mo = self.get().invoke_elem(bpi)
# print ("The footprint is %d" %mo.child_get("total-footprint"))
if(mo.results_status() == "failed"):
raise NAException(xo.sprintf())
tag = mo.child_get_string("next-tag")
while True:
# # get list of volume-info attributes
volattrs = dict()
volattrs = mo.child_get("attributes-list")
if (volattrs == None):
break
for part in volattrs.children_get():
vol_name = part.child_get_string("volume")
footprint= part.child_get_string("total-footprint")
if vol_name :
footprintdict[vol_name]=footprint
#The footprintlist should be taken to process the output
# get next set of records
bpi = NaElement("volume-footprint-get-iter")
bpi.child_add_string("max-records", mrecs)
bpi.child_add_string("tag", tag)
mo = self.get().invoke_elem(bpi)
if(mo.results_status() == "failed"):
raise NAException(xo.sprintf())
# Get next tag which indicates if there are more records
tag = mo.child_get_string("next-tag")
if (tag == None):
break
junction_path_map = dict()
comment_field_map = dict()
vol_usage_map = dict()
vol_dedup_saved = dict()
vol_dedup_shared = dict()
# Get First Set of mrecs (100) records
mrecs = "100"
api = NaElement("volume-get-iter")
api.child_add_string("max-records", mrecs)
xo = self.get().invoke_elem(api)
if(xo.results_status() == "failed"):
raise NAException(xo.sprintf())
# Set up to get next set of records
tag = xo.child_get_string("next-tag")
# Add vollist results to list and then Get Following set of records
# Break from loop when the vollist is None or next-tag is None
while True:
# get list of volume-info attributes
vollist = dict()
vollist = xo.child_get("attributes-list")
if (vollist == None):
break
# loop thru list of volumes to get specific volume attribute data
for tattr in vollist.children_get():
vol_id_attrs = tattr.child_get("volume-id-attributes")
if vol_id_attrs:
volume_name = vol_id_attrs.child_get_string("name")
junct_path = vol_id_attrs.child_get_string("junction-path")
# print ("\n junction path is %s " %junct_path)
comment_field = vol_id_attrs.child_get_string("comment")
# store junction_path for later
junction_path_map[volume_name] = junct_path
# if the comment field is empty, just put UNKNOWN as the value
if comment_field:
comment_field_map[volume_name] = comment_field
else:
comment_field_map[volume_name] = "UNKNOWN"
vol_space_attrs = tattr.child_get( "volume-space-attributes" );
if vol_space_attrs:
vol_usage = vol_space_attrs.child_get_string( "size-used" );
if vol_usage:
vol_usage_map[volume_name] = vol_usage;
#print "DEBUG: vol usage: $volume_name $vol_usage_map{$volume_name}\n";
dedup_saved=""
vol_sis_attrs = tattr.child_get( "volume-sis-attributes" )
if vol_sis_attrs:
dedup_saved = vol_sis_attrs.child_get_string( "percentage-total-space-saved" )
dedup_shared = vol_sis_attrs.child_get_string( "deduplication-space-shared" )
if dedup_saved:
vol_dedup_saved[volume_name] = dedup_saved
vol_dedup_shared[volume_name] = dedup_shared
# get next set of records
api = NaElement("volume-get-iter")
api.child_add_string("max-records", mrecs)
api.child_add_string("tag", tag)
xo = self.get().invoke_elem(api)
if(xo.results_status() == "failed"):
raise NAException(xo.sprintf())
# Get next tag which indicates if there are more records
tag = xo.child_get_string("next-tag")
if (tag == None):
break
#----------------------------------------
# create report header
#----------------------------------------
list = "\nList FlexClones\n";
#123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
list += "%-25s %-30s %-20s " % ("Parent Volume", "Parent-Snapshot", "FlexClone");
list += "%15s" % "Parent Space Con";
list += "%25s" % "Flexclone Space Con";
# list += "%15s" % "Split Est";
list += "%30s" % "Flexclone % of Parent";
list += " %s \n" % "Linux-Clone-path";
list += "---------------------------------------------------------------------------------------";
list += "---------------------------------------------------------------------------------------------------\n";
#----------------------------------------
# get FlexCone info iteratively - it will return a list
#----------------------------------------
# Get First Set of mrecs (100) records
mrecs = "100"
api = NaElement("volume-clone-get-iter")
api.child_add_string("max-records", mrecs)
xo = self.get().invoke_elem(api)
if(xo.results_status() == "failed"):
raise NAException(xo.sprintf())
tag = xo.child_get_string("next-tag")
# Add vollist results to list and then Get Following set of records
# Break from loop when the vollist is None or next-tag is None
while True:
# # get list of volume-info attributes
#vollist is the orignal volume list - all volumes
vollist = dict()
vollist = xo.child_get("attributes-list")
if (vollist == None):
break
# for each clone entry
# Use the P4 list here to compare against
clone_name="somestuffyyyyy"
for cpart in clonelist:
cname = cpart['client'][len(FLEX_CLONE):]
for vol_data in vollist.children_get():
clone_name = vol_data.child_get_string( "volume");
if str(cname)==str(clone_name):
volume_name = vol_data.child_get_string( "parent-volume" );
clone_name = vol_data.child_get_string( "volume" );
snapshot = vol_data.child_get_string( "parent-snapshot");
#flexclone_used = vol_data.child_get_string( "used" );
# split_est = vol_data.child_get_string( "split-estimate" );
comment_field = comment_field_map[clone_name];
if comment_field == "":
comment_field = "USER_UNKNOWN"
#
# # parent volume: space used - represent data used in MB
vol_usage = footprintdict[volume_name]
parent_footprint = float(vol_usage)/1024/1024
#
# # FlexClone volume: space used - represent data used in MB
clone_usage = footprintdict[clone_name]
clone_footprint = float(clone_usage) / 1024 / 1024;
# Find how much of the parent volume size is consumed by the clone
pctofparent = float(clone_footprint/parent_footprint)*100
#
# determine juction-path info
jpath = str(self.mount) + vol_data.child_get_string( "junction-path" );
# test if the value returned correctly
if jpath:
# # perfect the look up worked correctly
dummyval = 0;
elif junction_path_map[clone_name]:
# # ok lookup didn't work, but it was found by method #2
jpath = junction_path_map[clone_name]
else:
# no junction path found
jpath = "Not Mounted";
# print results
list += "%-25s %-30s %-20s " % (volume_name, snapshot, clone_name);
list += "%11.2f MB " % parent_footprint;
list += "%20.2f MB " % clone_footprint;
# list += "%12.2f MB " % split_est;
# list += "%19.2f MB" % clone_footprint;
list += " %21.2f" % pctofparent;
list += "%";
# list += " (%5.2f" % compression; print "%)";
# list += "%15s" % comment_field;
list += " %s\n" % jpath;
# get next set of records
api = NaElement("volume-get-iter")
api.child_add_string("max-records", mrecs)
api.child_add_string("tag", tag)
xo = self.get().invoke_elem(api)
if(xo.results_status() == "failed"):
raise NAException(xo.sprintf())
# Get next tag which indicates if there are more records
tag = xo.child_get_string("next-tag")
if (tag == None):
break
return list
# # Prints out list of clones with their corresponding parent volume and parent snapshot
def clist(self):
# Get first set number of records defined by mrecs
mrecs = "100"
api = NaElement("volume-clone-get-iter")
api.child_add_string("max-records", mrecs)
xo = self.get().invoke_elem(api)
if(xo.results_status() == "failed"):
raise NAException(xo.sprintf())
# Set up to get next set of records
tag = xo.child_get_string("next-tag")
print("\n")
# Need to go thru this loop at least once before checking for next tag
while True:
# Get list of snapshot-info attributes
clonelist = dict()
clonelist = xo.child_get("attributes-list")
if (clonelist == None):
break
for clone in clonelist.children_get():
print ("clone %s:%s:%s" % (clone.child_get_string("parent-volume"), clone.child_get_string("parent-snapshot"), clone.child_get_string("volume")))
# Get next set of records
api = NaElement("volume-clone-get-iter")
api.child_add_string("max-records", mrecs)
api.child_add_string("tag", tag)
xo = self.get().invoke_elem(api)
if(xo.results_status() == "failed"):
raise NAException(xo.sprintf())
# Tag indicates if there are more records
tag = xo.child_get_string("next-tag")
# Break if no more records
if (tag == None):
break
class NAException(Exception):
def __init__(self, e):
self.error = e
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Perforce connection as 'flex' user
# ---------------------------------------------------------------------------
class P4Flex:
def __init__(self):
self.port = config.get("p4.port")
self.user = config.get("p4.user")
self.passwd = config.get("p4.passwd")
self.osuser = config.get("p4.osuser")
def getP4(self):
p4 = P4()
p4.user = self.user
p4.port = self.port
p4.connect()
p4.password = self.passwd
p4.run_login()
return p4
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Parse the Broker's request and provide access to the user's environment
# ---------------------------------------------------------------------------
class Broker:
def __init__(self):
self.args = {}
# Comment out stdin and uncomment block below for debug.
lines = sys.stdin.readlines()
for line in lines:
parts = line.split(": ")
self.args[parts[0]] = parts[1].rstrip()
def getP4(self):
p4 = P4()
p4.user = self.args['user']
p4.port = self.args['clientPort']
p4.client = self.args['workspace']
p4.connect()
# Use flex account to login user
s4 = P4Flex().getP4()
s4.run_login(self.args['user'])
return p4
def getPort(self):
return self.args['clientPort']
def getUser(self):
return self.args['user']
def getClient(self):
return self.args['workspace']
def getCommand(self):
if 'Arg0' in self.args:
return self.args['Arg0']
else:
return None
def getOptions(self):
c = 1
opts = []
while True:
key = 'Arg' + str(c)
c += 1
if key in self.args:
opts.append(self.args[key])
else:
break
join = ""
list = []
for opt in opts:
if join:
opt = join + opt
join = ""
if opt.startswith('-') and len(opt) == 2:
join = opt
else:
list.append(opt)
if join:
list.append(opt)
return list
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Handle Broker request and invoke the registered 'flex' command
# ---------------------------------------------------------------------------
class Flex:
def __init__(self):
# Process Broker arguments
self.call = Broker()
self.command = self.call.getCommand()
self.args = self.call.args
self.opts = self.call.getOptions()
# Build table for function pointers
self.cmdFn = {
'volume': self.volume,
'volumes': self.volumes,
'lv': self.volumes,
'snapshot': self.snapshot,
'snapshots': self.snapshots,
'ls': self.snapshots,
'clone': self.clone,
'clones': self.clones,
'lc': self.clones,
'help': self.help
};
if not self.isnfs(NaFlex().mount):
print("action: REJECT")
print("message: \"%s\"" % "NetApp nfs volume is not mounted.")
return
# Call command
if self.command in self.cmdFn:
self.cmdFn[self.command]()
else:
self.usage()
def volume(self):
# Check for 'admin' permission
# NetApp connection
netapp = NaFlex()
if not self.permission():
print("action: REJECT")
print("message: \"%s\"" % "You don't have permission for this operation.")
return
ontapversion = netapp.getontapversion()
# Get options (set defaults)
vname = ""
size = "1G"
user = self.call.getUser()
uid = pwd.getpwnam(user).pw_uid
gid = ""
junction_path = ""
for o in self.opts:
if o.startswith('-d'):
del_name = o[2:]
self.vol_del(del_name)
return
if o.startswith('-s'):
size = o[2:]
if o.startswith('-j'):
junction_path = o[2:]
if o.startswith('-u'):
uid = pwd.getpwnam(o[2:]).pw_uid
if o.startswith('-g'):
gid = grp.getgrnam(o[2:]).gr_gid
else:
vname = o
if not gid:
if uid:
gid = pwd.getpwuid(uid).pw_gid
else:
p4group = config.get("p4.osuser")
gid = pwd.getpwnam(p4group).pw_gid
if not vname:
print("action: REJECT")
print("message: \"%s\"" % "No flex volume name provided")
return
if not junction_path:
junction_path = "/" + vname
# Create NetApp volume
try:
#with added cDOT 9 functionality
netapp.create(vname, size,junction_path, uid, gid)
# Set file ownership of newly created volume.
user = self.call.getUser()
path = netapp.mount + junction_path
#Add in autogen of .p4config for new P4 Volume
self.p4config(path, vname)
# self.chown(user, path)
msg = vname + ". Mounted on " + path + "."
if "9." not in ontapversion:
self.chown(user, path)
msg += ("\n If necessary, run 'sudo chown -R %s:%s %s ' to set correct volume ownership" %(user, user, path))
# msg += ("\n Run 'sudo chown -R %s:%s %s ' to set correct volume ownership" %(user, user, path))
print("action: RESPOND")
print("message: \"Created flex volume %s\"" % msg)
except NAException as e:
print("action: RESPOND")
error = '\nNetApp Error: ' + e.error
print("message: \"%s\"" % error)
except Exception as e:
# If got unexpected error, delete the volume created
error = "Unexpected Error: " + str(e)
try:
netapp.delete(vname)
except NAException as e:
error += '\nNetApp Error: ' + e.error
print("action: RESPOND")
print("message: \"%s\"" % error)
def volumes(self):
# NetApp connection
netapp = NaFlex()
# List the volumes
try:
list = "P4 Flex Volumes \n"
vols = netapp.vlist()
for v in vols:
list += "volume '"
list += v.child_get_string("name") + "' "
list += "\n"
print("action: RESPOND")
print("message: \"%s\"" % list)
except NAException as e:
print("action: RESPOND")
error = '\nNetApp Error: ' + e.error
print("message: \"%s\"" % error)
# Builds a new FLEX_SNAP workspace to create the snapshot
def snapshot(self):
# Get options (set defaults)
vname = ""
sname = ""
from_client_name = self.call.getClient()
for o in self.opts:
if o.startswith('-V'):
vname = o[2:]
if o.startswith('-d'):
dname=o[2:]
self.snap_del(vname,dname)
return
if o.startswith('-c'):
from_client_name = o[2:]
else:
sname = o
if not vname:
print("action: REJECT")
print("message: \"%s\"" % "No flex volume name provided")
return
if not sname:
print("action: REJECT")
print("message: \"%s\"" % "No flex clone name provided")
return
if ':' in sname:
print("action: REJECT")
print("message: \"%s\"" % "Flex clone name must not use ':'")
return
if not self.permission():
print("action: REJECT")
print("message: \"%s\"" % "You don't have permission for this operation.")
return
# NetApp/Perforce connection as Flex
netapp = NaFlex()
p4 = P4Flex().getP4()
# Create Perforce workspace for snapshot
try:
from_client = p4.fetch_client(from_client_name)
root = from_client['Root']
# Clone client workspace
flex_client_name = FLEX_SNAP + vname + ':' + sname
p4.client = flex_client_name
flex_client = p4.fetch_client("-t", from_client_name, flex_client_name)
# Set workspace options: root to mounted volume
path = netapp.mount + "/" + vname
flex_client['Root'] = path
flex_client['Host'] = ""
#flex_client['Options'] = flex_client['Options'].replace(" unlocked", " locked")
p4.save_client(flex_client)
# Populate have list
p4.run_sync("-k", "//" + flex_client_name + "/...@" + from_client_name)
print("action: RESPOND")
print("message: \"Created flex snapshot %s\"" % sname)
except P4Exception:
print("action: RESPOND")
error = '\n'.join(p4.errors)
error += '\n'.join(p4.warnings)
print("message: \"%s\"" % error)
sys.exit()
finally:
p4.disconnect()
# NetApp/Perforce connection as Flex
netapp = NaFlex()
p4 = P4Flex().getP4()
# Create NetApp snapshot
try:
netapp.snapshot(vname, sname)
except NAException as e:
print("action: RESPOND")
error = '\nNetApp Error: ' + e.error
print("message: \"%s\"" % error)
def snap_del(self, vname,snapname):
if not snapname:
print("action: REJECT")
print("message: \"%s\"" % "No flex snapshot name provided")
return
if not vname:
print("action: REJECT")
print("message: \"%s\"" % "No volume name provided")
return
# NetApp/Perforce connection as Flex
netapp = NaFlex()
p4 = P4Flex().getP4()
try:
graboutput=str(netapp.snapshot_delete(vname,snapname))
if "ERROR" in graboutput:
errstring = ("\n You must delete clones associcated with the snapshot %s first\n" %snapname)
raise NAException(errstring + graboutput)
except NAException as e:
print("action: REJECT")
error = '\nNetApp Error: ' + e.error
print("message: \"%s\"" % error)
return
except Exception as e:
print("action: REJECT")
error = '\nUnexpected error: ' + str(e)
print("message: \"%s\"" % error)
return
#print ("\n NetApp Error : %s" %graboutput)
# Delete Perforce workspaces
try:
client_name = FLEX_SNAP + vname + ":" + snapname
p4.run_client("-f", "-d", client_name)
print("action: RESPOND")
message = "Deleted snapshot %s\n" % snapname
print("message: \"%s\"" % message)
except P4Exception:
print("action: RESPOND")
error = '\n'.join(p4.errors)
error += '\n'.join(p4.warnings)
finally:
p4.disconnect()
# If client delete succeed, Delete NetApp snapshot
def permission(self):
p4 = self.call.getP4()
try:
# Perforce connection as Caller
# p4 = self.call.getP4()
for p in p4.run_protects():
if (p['perm'] == 'admin') or (p['perm'] == 'super'):
return True
return False
except P4Exception:
print("action: RESPOND")
error = '\n'.join(p4.errors)
error += '\n'.join(p4.warnings)
print("message: \"%s\"" % error)
finally:
p4.disconnect()
def snapshots(self):
try:
# Perforce connection as Caller
p4 = self.call.getP4()
snapshots = p4.run_clients("-e", FLEX_SNAP + "*")
list = ""
list += "%-25s %-30s %-45s \n" %("snapshot", "base-volume", "base-volume-path")
for v in snapshots:
name = v['client'][len(FLEX_SNAP):]
part = name.split(':', 1)
list += "%-25s %-30s %-45s \n" %(part[1], part[0], v['Root'])
print("action: RESPOND")
print("message: \"%s\"" % list)
except P4Exception:
print("action: RESPOND")
error = '\n'.join(p4.errors)
error += '\n'.join(p4.warnings)
print("message: \"%s\"" % error)
finally:
p4.disconnect()
# Clones a new FLEX_CLONE workspace from the parent flexSnap_
def clone(self):
# Get options
vname = ""
sname = ""
cname = ""
# Set directory ownership
user = self.call.getUser()
uid = pwd.getpwnam(user).pw_uid
gid = ""
junction_path = ""
all=False
for o in self.opts:
if o.startswith('-d'):
del_name = o[2:]
self.clone_del(del_name)
return
if o.startswith('-V'):
vname = o[2:]
if o.startswith('-S'):
sname = o[2:]
if o.startswith('-j'):
junction_path = o[2:]
if o.startswith('-u'):
uid = pwd.getpwnam(o[2:]).pw_uid
if o.startswith('-g'):
gid = grp.getgrnam(o[2:]).gr_gid
else:
cname = o
if not gid:
if uid:
gid = pwd.getpwuid(uid).pw_gid
else:
p4group = config.get("p4.osuser")
gid = pwd.getpwnam(p4group).pw_gid
if not vname:
print("action: REJECT")
print("message: \"%s\"" % "No flex volume name provided")
return
if not sname:
print("action: REJECT")
print("message: \"%s\"" % "No flex snapshot name provided")
return
if not cname:
print("action: REJECT")
print("message: \"%s\"" % "No flex clone name provided")
return
if ':' in cname:
print("action: REJECT")
print("message: \"%s\"" % "Flex clone name must not use ':'")
return
if not junction_path:
junction_path = netapp.mount + cname
# NetApp/Perforce connection as Caller
netapp = NaFlex()
p4 = self.call.getP4()
path = netapp.mount + "/" + cname
# Create NetApp clone from snapshot
ontapversion = netapp.getontapversion()
# Generate the clone
try:
netapp.clone(cname, sname, vname, uid, gid,junction_path)
except NAException as e:
print("action: RESPOND")
error = '\nNetApp Error: ' + e.error
print("message: \"%s\"" % error)
try:
# Verify parent client exists
parent_client_name = FLEX_SNAP + vname + ":" + sname
list = p4.run_clients("-e", parent_client_name)
if len(list) != 1:
print("action: REJECT")
print("message: \"Flex parent %s does not exist.\"" % parent_client_name)
return
# Clone client workspace
clone_client_name = FLEX_CLONE + cname
p4.client = clone_client_name
clone_client = p4.fetch_client("-t", parent_client_name, clone_client_name)
clone_client['Root'] = junction_path
p4.save_client(clone_client)
# Generate P4CONFIG file based on the clone
path=netapp.mount+junction_path
self.p4config(path, clone_client_name)
# Populate have list
p4.run_sync("-k", "//" + clone_client_name + "/...@" + parent_client_name)
msg = clone_client_name + ". Mounted on " + junction_path + "."
if "9." not in ontapversion:
self.chown(user, path)
msg += ("\n If necessary, run 'sudo chown -R %s:%s %s ' to set correct volume ownership" %(user, user, path))
print("action: RESPOND")
print("message: \"Created flex clone client %s\"" % msg)
except P4Exception:
print("action: RESPOND")
error = '\n'.join(p4.errors)
error += '\n'.join(p4.warnings)
print("message: \"%s\"" % error)
finally:
p4.disconnect()
def chown(self, user, path):
# Checks need to be done in case of failure
uid = pwd.getpwnam(user).pw_uid
gid = pwd.getpwnam(user).pw_gid
cmd=['sudo', 'chown', '-R', str(uid)+':'+str(gid), path]
subprocess.call(cmd, shell=True)
def isnfs(self, path):
# Check if path is nfs mounted.
fstype = subprocess.check_output('stat -f -L -c %T ' + path, shell=True)
if (fstype == 'nfs\n'):
return True
return False
def clones(self):
# Get options
all = False
for o in self.opts:
if o == "-a":
all = True
try:
# Perforce connection as Caller
p4 = self.call.getP4()
clones = []
if all:
clones = p4.run_clients("-e", FLEX_CLONE + "*")
else:
user = self.call.getUser()
clones = p4.run_clients("-e", FLEX_CLONE + "*", "-u", user)
list = ""
if len(clones)!=0:
netapp = NaFlex()
message=""
try:
print("action: RESPOND")
message += netapp.list_flexclones(clones)
print("message: \"%s\"" % message)
except P4Exception:
print("action: RESPOND")
error = '\n'.join(p4.errors)
error += '\n'.join(p4.warnings)
message += error
print("error message: \"%s\"" % message)
else:
print("action: RESPOND")
message = "No Clones Found"
print("message: \"%s\"" % message)
except P4Exception:
print("action: RESPOND")
error = '\n'.join(p4.errors)
error += '\n'.join(p4.warnings)
print("message: \"%s\"" % error)
finally:
p4.disconnect()
def clone_del(self, cname):
if not cname:
print("action: REJECT")
print("message: \"%s\"" % "No Flex clone name provided")
return
# NetApp/Perforce connection as Caller
netapp = NaFlex()
p4 = self.call.getP4()
# Delete Perforce workspace
try:
client_name = FLEX_CLONE + cname
p4.run_client("-d", client_name)
print("action: REJECT")
print("message: \"Deleted Flex clone %s\"" % cname)
except P4Exception:
print("action: REJECT")
error = '\n'.join(p4.errors)
error += '\n'.join(p4.warnings)
print("message: \"%s\"" % error)
return
finally:
p4.disconnect()
# If client delete succeed, Delete NetApp clone
try:
netapp.delete(cname)
except NAException as e:
print("action: REJECT")
error = '\nNetApp Error: ' + e.error
print("message: \"%s\"" % error)
return
except Exception as e:
print("action: REJECT")
error = '\nUnexpected error: ' + str(e)
print("message: \"%s\"" % error)
return
def vol_del(self, vname):
graboutput=""
if not vname:
print("action: REJECT")
print("message: \"%s\"" % "No flex volume name provided")
return
# NetApp/Perforce connection as Flex
netapp = NaFlex()
#First Check to see if any clones off the volume exist. If they do, exit and don't destroy anything
try:
if netapp.ckclone(vname)=="True":
raise NAException ("\n The master volume %s has one or more clones which must be deleted first" %vname)
except NAException as e:
print("action: REJECT")
error = '\nNetApp Error: ' + e.error
print("message: \"%s\"" % error)
return
except Exception as e:
print("action: REJECT")
error = '\nUnexpected error: ' + str(e)
print("message:\"%s\"" % error)
return
try:
graboutput=str(netapp.delete(vname))
if "error" in graboutput.lower():
errstring = ("\n You must delete clones associcated with the master volume %s first\n" %vname)
raise NAException(errstring + graboutput)
except NAException as e:
print("action: REJECT")
error = '\nNetApp Error: ' + e.error
print("message: \"%s\"" % error)
return
except Exception as e:
print("action: REJECT")
error = '\nUnexpected error: ' + str(e)
print("message:\"%s\"" % error)
return
# Delete Perforce workspaces
# should only enter if upper part passed
p4 = P4Flex().getP4()
try:
clones = p4.run_clients("-e", FLEX_SNAP + vname + "*")
list = ""
for c in clones:
client_name = c['client']
p4.run_client("-f", "-d", client_name)
list += " deleted client: %s\n" % client_name
#
print("action: RESPOND")
message = "Deleted Flex volume %s\n" % vname
message += list
print("message: \"%s\"" % message)
except P4Exception:
print("action: RESPOND")
error = '\n'.join(p4.errors)
error += '\n'.join(p4.warnings)
print("message: \"%s\"" % error)
finally:
p4.disconnect()
def p4config(self, path, client):
p4config = os.getenv('P4CONFIG', '.p4config')
p4port = self.call.getPort()
p4user = self.call.getUser()
fh = open(path + "/.p4config", "wt")
try:
if fh:
fh.write("# Generated by p4 flex.\n");
fh.write("P4PORT=" + p4port + "\n");
fh.write("P4USER=" + p4user + "\n");
fh.write("P4CLIENT=" + client + "\n");
finally:
fh.close()
fh.close()
def help(self):
help = (
"P4 FlexClone Usage Information\n\n"
" P4 Broker functions for creating and managing NetApp FlexClones\n\n"
"\n"
"SYNOPSIS\n"
" p4 flex [command] [command options]\n"
"\n"
"DESCRIPTION\n"
" Create Volume\n"
" p4 flex volume -s <vol size[M, G]> -j <junction_path> [-u user] [-g group] <volume name>\n"
" Delete Volume\n"
" p4 flex volume -d <volume name>\n"
" List Volumes\n"
" p4 flex volumes \n"
" p4 flex lv\n"
"\n"
" Create Snapshot\n"
" p4 flex snapshot -V <volume volume> [-c client] <snapshot name>\n"
" Delete Snapshot\n"
" p4 flex snapshot -V <volume name> -d <snapshot name>\n"
" List Available Snapshots (with parent volume)\n"
" p4 flex snapshots -V <volume name (optional)>\n"
" p4 flex ls -V <volume name (optional)>\n"
"\n"
" Create FlexClone (aka clone)\n"
" p4 flex clone -V <volume> -S <snapshot name> -j <junction_path> [-u user] [-g group] <clone name>\n"
" Delete FlexClone\n"
" p4 flex clone -d <clone name>\n"
" List FlexClones\n"
" p4 flex clones -V <volume name (optional)> [-a]\n"
" p4 flex lc -V <volume name (optional)> [-a]\n"
"\n"
)
print("action: RESPOND")
print("message: \"%s\"" % help)
def usage(self):
usage = (
"Usage: flex { volume(s) | snapshot(s) | clone(s) }\n"
"Missing/wrong number of arguments."
)
print("action: RESPOND")
print("message: \"%s\"" % usage)
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# MAIN
# ---------------------------------------------------------------------------
if __name__ == '__main__':
flex = Flex()
| # | Change | User | Description | Committed | |
|---|---|---|---|---|---|
| #9 | 24146 | Paul Allen | Copy changes from punithc/p4flex | ||
| #8 | 20227 | netapp | small change in list_flexclones function to show the flexclone path in Linux, versus just the junction path. | ||
| #7 | 20171 | netapp | flex.py has new code that uses volume footprints instead of old cDOT8 code when printing out the list of clones. | ||
| #6 | 20103 | netapp |
Latest code - Aug 15, 2016. Changes made to address : 1. Added in functionality for snapshot delete ('snapshot -V volname -d snapshotsname') 2. Corrected volume delete to not allow for Helix database volume delete, unless all clones are first removed 3. 'p4 flex clones' now produces full table showing Parent Volume, Parent-Snapshot, FlexClone name, Parent volume space consumed, FlexClone volume space consumed, FlexClone acutal (space versus parent volume), and the Clone-path 4. Flex Clones now thinly provisioned to allow for storage space savings 5. Code change to process P4 first when taking snapshots, and then follow with NetApp step versus previous reverse order |
||
| #5 | 19802 | netapp | Corrected empty clone list error. | ||
| #4 | 19800 | netapp |
Many changes - 1 ) Snapshot Delete Capability 2) Sizing of FlexClones versus original volume reported showing space saved 3) Check for volume Clones before volume deletion 4) Output of chown command when volumes created by user |
||
| #3 | 19636 | netapp | Reverting back to Agnes' original | ||
| #2 | 19474 | netapp |
Uploaded new files for project. Enhancements include: 1. Added in snapshot deletion capability 2. Created separate user clone mount directory, which can be different from the base directory 3. Cleaned up error handling for all p4 flex calls, so errors are handled with understandable output, versus program crashes 4. Added in ability to specify volume owner when creating a volume with �p4 flex volume �s �u�. |
||
| #1 | 18997 | netapp |
Rename/move file(s) Project name change: moving //guest/netapp/p4flexclone/main/… to //guest/netapp/p4flex/main/… |
||
| //guest/netapp/p4flexclone/main/demo/flex.py | |||||
| #36 | 15417 | Paul Allen | Remove client lock. | ||
| #35 | 15070 | Paul Allen | spelling | ||
| #34 | 15068 | Paul Allen | Allow locking on snapshot workspaces and use '-f' on volume delete. | ||
| #33 | 15067 | Paul Allen |
Disable workspace locking for demo. Better solution is for user 'flex' to delete with a '-f', but check ownership first. |
||
| #32 | 15066 | Paul Allen | Update create clone message. | ||
| #31 | 15065 | Paul Allen | Back out changelist 15064 | ||
| #30 | 15064 | Paul Allen | chown fix? | ||
| #29 | 15063 | Paul Allen | Remove description from 'flex volumes' output. | ||
| #28 | 15062 | Paul Allen |
Use Flex user to login current user (needs super) Minor fix to help |
||
| #27 | 15027 | agnesj | Added a check if NetApp volume is mounted, removed test function, and modified -P to -S for flex clone command. | ||
| #26 | 14926 | Paul Allen | Move chown operations to method and use system call to 'chown -R' | ||
| #25 | 14925 | Paul Allen | Generate a P4CONFIG file in the mounted clone's volume. | ||
| #24 | 14924 | Paul Allen | Set workspace 'Root' to mount location for clone and snapshot. | ||
| #23 | 14923 | Paul Allen |
Show Owner when listing clones and volume when listing snapshots. Includes a tidy up on all listing commands. |
||
| #22 | 14921 | Paul Allen |
#review-14917 Verify delete permission for volume/clone. - Only a Perforce 'admin' can delete a volume - Only a Perforce 'admin' and the clone's owner can delete a clone. Use Perforce client workspace based permissions to enforce rules. If the client delete succeeds assume all is ok to delete the volume/clone. Example error: bob$ p4 flex clone -d clone_2_1_a Locked client 'flexClone:clone_2_1_a' owned by 'pallen'; use -f to force update. |
||
| #21 | 14895 | agnesj |
Modified volume function such that when creating a volume the owner of volume is modified. Also modified vol_del and clone_del functions to check for ownership of volume. Only owners of volumes are allowed to delete. |
||
| #20 | 14881 | agnesj |
Added checks in chown and volume functions. And added location of newly created volume in message. |
||
| #19 | 14180 | Paul Allen |
Set file ownership for clone's mounted files. + Minor fix to clone delete |
||
| #18 | 14179 | Paul Allen | Add support for 'flex volumes' command. | ||
| #17 | 14178 | agnesj | Modified NetApp delete to check if volume is a root volume (vserver or node) and to not allow p4 user to delete. | ||
| #16 | 14141 | agnesj | Added definition functions for volume list, snapshot list based on volume name, all snapshot list, clone list | ||
| #15 | 13997 | Paul Allen | Stop on error (don't delete clients if vol delete fails). | ||
| #14 | 13996 | Paul Allen | Added -s to volume for size. | ||
| #13 | 13993 | Paul Allen |
Added NetApp commands to Flex. Example: 4007::ws$ p4 flex volume vol1 Created flex volume vol1 4007::ws$ p4 flex snapshot -V vol1 snapshot1 Created flex snapshot snapshot1 4007::ws$ p4 flex snapshot -V vol1 snapshot2 Created flex snapshot snapshot2 4007::ws$ p4 flex snapshots snapshot vol1:snapshot1 root /home/pallen/flexclone/demo/ws snapshot vol1:snapshot2 root /home/pallen/flexclone/demo/ws 4007::ws$ p4 flex clone -V vol1 -P snapshot1 clone_A Created flex clone client flexClone:clone_A 4007::ws$ p4 flex clone -V vol1 -P snapshot1 clone_B Created flex clone client flexClone:clone_B 4007::ws$ p4 flex clones Clone clone_A root /home/pallen/flexclone/demo Clone clone_B root /home/pallen/flexclone/demo 4007::ws$ p4 clients Client flexClone:clone_A 2015/06/17 root /home/pallen/flexclone/demo 'Created by pallen. ' Client flexClone:clone_B 2015/06/17 root /home/pallen/flexclone/demo 'Created by pallen. ' Client flexSnap:vol1:snapshot1 2015/06/17 root /home/pallen/flexclone/demo/ws 'Created by flex. ' Client flexSnap:vol1:snapshot2 2015/06/17 root /home/pallen/flexclone/demo/ws 'Created by flex. ' Client pallen-ws 2015/06/10 root /home/pallen/flexclone/demo/ws 'Created by pallen. ' 4007::ws$ p4 flex clone -d clone_B Deleted Flex clone clone_B 4007::ws$ p4 flex clone -d clone_A Deleted Flex clone clone_A 4007::ws$ p4 flex volume -d vol1 Deleted Flex volume vol1 deleted client: flexSnap:vol1:snapshot1 deleted client: flexSnap:vol1:snapshot2 4007::ws$ p4 clients Client pallen-ws 2015/06/10 root /home/pallen/flexclone/demo/ws 'Created by pallen. ' |
||
| #12 | 13988 | Paul Allen | Tidy up formatting and imports. | ||
| #11 | 13985 | agnesj |
Added definitions: clone, snapshot, and delete Modified test functions to test each of the added functions |
||
| #10 | 13867 | agnesj | Made corrections to NaFlex Class to pull info from flex.cfg file | ||
| #9 | 13859 | Paul Allen | Config details for cloud demo + NaServer test | ||
| #8 | 13796 | Paul Allen |
Moved options to a configuration file. - The config file 'flex.cfg' MUST be located in the same dir as the script - Provided a sample 'in.txt' file to simulate a broker call (used for debug only) |
||
| #7 | 13791 | Paul Allen | Changed 'volume' to 'snapshot' and minor fix to for auth/login. | ||
| #6 | 13702 | Paul Allen | Added Permission check for p4 flex volume command. | ||
| #5 | 13671 | Paul Allen |
Added '-a' option to flex clones command. Tidy up and hide flexVol/flexClone prefix from user. |
||
| #4 | 13669 | Paul Allen | Added '-d' option to delete flex clone client. | ||
| #3 | 13668 | Paul Allen | Added Perforce functionality for volume(s) and clone(s) commands. | ||
| #2 | 13628 | Paul Allen | set executable | ||
| #1 | 13627 | Paul Allen | New Python script with Implemented help/usage | ||