Source code for geni.portal

# Copyright (c) 2014-2019 The University of Utah and Barnstormer Softworks, Ltd.

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

"""Library for dealing with scripts that are run in the context of a portal."""

from __future__ import absolute_import

import sys
import os
import atexit
import warnings
import json
import argparse
from argparse import Namespace
import logging
import os.path

import six

from .rspec import igext
from .rspec import pgmanifest
from .rspec.pg import Request

# Default to something sane for our package
LOG = logging.getLogger('geni.portal')

dodebug = False
debugLogHandler = None
if 'GENILIB_PORTAL_DEBUG' in os.environ:
    dodebug = True
    debugLogHandler = logging.StreamHandler(stream=sys.stderr)
    debugLogHandler.setFormatter(logging.Formatter(
        fmt="%(levelname)s:%(name)s:%(lineno)s:%(funcName)s: %(message)s "))
    LOG.addHandler(debugLogHandler)
    LOG.setLevel(logging.DEBUG)
else:
    LOG.addHandler(logging.NullHandler())

[docs]class DictNamespace(dict,Namespace): """ A simple class that is similar to argparse.Namespace, but is also an iterable dictionary, and allows key/value deletion. """ def __setattr__(self,attr,value): self.__setitem__(attr,value) def __getattr__(self,attr): if attr in self.keys(): return self.__getitem__(attr) else: return dict.__getattribute__(self,attr) def __delattr__(self,attr): if attr in self.keys(): return self.__delitem__(attr) else: return dict.__delattribute__(self,attr)
[docs]def parseBool(b): """ Extract a bool from `b` as a string (any type), or by using `bool()`. """ if isinstance(b,six.string_types): if b == "False" or b == "false": return False elif b == "True" or b == "true": return True return bool(b)
[docs]class ParameterType (object): """Parameter types understood by Context.defineParameter().""" INTEGER = "integer" #: Simple integer STRING = "string" #: Any string BOOLEAN = "boolean" #: True/False IMAGE = "image" #: URN specifying a particular image AGGREGATE = "aggregate" #: URN specifying an Aggregate Manger NODETYPE = "nodetype" #: String specifying a type of node BANDWIDTH = "bandwidth" #: Floating-point number to be used for bandwidth LATENCY = "latency" #: Floating-point number to be used for latency SIZE = "size" #: Integer for size (eg. MB, GB, etc.) PUBKEY = "pubkey" #: An RSA public key. LOSSRATE = "lossrate" #: Floating-point number 0.0 <= N < 1.0 # Dynamically created drop down menus of Powder specific resources FIXEDENDPOINT = "fixedendpoint" #: Fixed Endpoints BASESTATION = "basestation" #: Base Stations argparsemap = { INTEGER: int, STRING: str, BOOLEAN: parseBool, IMAGE: str, AGGREGATE: str, NODETYPE: str, BANDWIDTH: float, LATENCY: float, SIZE: int, PUBKEY: str, LOSSRATE: float, FIXEDENDPOINT: str, BASESTATION: str} # Other parameter types known to the system, but not exposed via the # defineParameter API. STRUCT = "struct" #: A StructParameter with member parameters; may be a JSON string that encodes a dict, or a raw dict.
[docs]class Parameter(object): """ A class containing the definition of a basic parameter where type is specified as a ParameterType field (e.g., not multi-valued, not a struct type). Do not instantiate this class directly; use the method wrappers in the Context class to declare parameters when possible. Parameters must have a defaultValue. They may have a list of legalValues. All values in the latter must be type-correct, and the former must be both type-correct and a member of _legalValues. A legalValues list may be a flat list of type-correct values, or a list of 2-tuples, where the first element of each tuple is a type-correct value, and the second element is a display name for frontends. """ def __init__(self,name,description,type,defaultValue,legalValues=None, longDescription=None,inputFieldHint="",groupId=None,hide=False, prefix="emulab.net.parameter.",inputConstraints=None, _skipInitialChecks=False): self.name = name self.description = description self.longDescription = longDescription self.inputFieldHint = inputFieldHint self.inputConstraints = inputConstraints self.type = type self.groupId = groupId self.hide = hide self.prefix = prefix self._value = None self._legalValues = legalValues or None self._defaultValue = defaultValue if not _skipInitialChecks: if self.legalValues: for x in self.legalValues: self._parseValue(x) self.setDefaultValue(defaultValue) LOG.debug(str(self)) def __repr__(self): return "%s(%s=%s,default=%s)" % ( self.__class__.__name__,self.name,self.value,self.defaultValue) # NB: this is here for naughty profiles that used to reach into # pc._parameters[paramName][field]; this is not intended to be a # general dict-like interface. def __getitem__(self,item): return self.__getattribute__(item) @property def defaultValue(self): """ Return the default value of this parameter. """ return self._defaultValue @property def legalValues(self): """ Returns None if there are no legal value restrictions. Otherwise, returns a list of legal values. (NB: self._legalValues itself may be either a list of strings, or a list of 2-tuples, where the actual value is the first element of the tuple, and the display hint is the second element.) """ if not self._legalValues: return None return [x if not isinstance(x, tuple) else x[0] for x in self._legalValues] @property def value(self): """ Returns this parameter's bound value, after values have been bound to this parameter's portal Context object. """ return self._value def _parseValue(self,value): """ Extracts/converts a parameter value of the correct type, if possible, from the supplied `value`. To support the variety of input formats of parameters, we support conversion from strings to the proper ParameterType; see ParameterType.argparsemap; see also other Parameter subclasses (StructParameter) and mixins (Multi), since those accept JSON strings. """ LOG.debug("%s(%s)" % (str(self.name),str(value))) if value == None: raise IllegalParameterValueError(value,self) nvalue = ParameterType.argparsemap[self.type](value) LOG.debug("%s(%s) -> %s" % (str(self.name),str(value),str(nvalue))) return nvalue def _checkValue(self,value): """ Checks that a type-correct value (e.g. extracted by _parseValue) is also a member of _legalValues. """ LOG.debug("%s(%s)" % (self.name,str(value))) if self.legalValues and value not in self.legalValues: raise IllegalParameterValueError(value,self) if value == None: raise IllegalParameterValueError(value,self)
[docs] def setValue(self,value): """ Invoked by `Context` to bind a value to this parameter. This both parses the value and checks it for legality. """ v = self._parseValue(value) self._checkValue(v) self._value = v return v
[docs] def setDefaultValue(self,defaultValue): """ Sets this parameter's default value, after invoking its _parseValue (type correctness) and _checkValue (constraint legality) methods. """ LOG.debug("%s(%s)" % (self.name,str(defaultValue))) # NB: the defaultValue may be a tuple present in self._legalValues; it # need not be the first value of a tuple. if type(defaultValue) == tuple and defaultValue in self._legalValues: defaultValue = defaultValue[0] v = self._parseValue(defaultValue) self._checkValue(v) self._defaultValue = v
[docs] def toParamdef(self): """ Converts this Parameter's metadata to a JSON dict used by the portal frontend. """ fields = [ "name","description","longDescription","inputFieldHint","type", "_legalValues","_defaultValue","groupId","hide", "inputConstraints" ] d = dict() for f in fields: fname = f if fname.startswith('_'): fname = fname[1:] d[fname] = getattr(self,f) return d
[docs] def validate(self): """ A wrapper that checks both the defaultValue and legalValues for type- and constraint-correctness. """ LOG.debug(self.name) self._parseValue(self.defaultValue) self._checkValue(self.defaultValue) if self.legalValues: for v in self.legalValues: self._parseValue(v)
[docs]class Multi(object): """ When this class is added to a subclass of Parameter as the *first* multiply-inherited class, the subclass will accept lists as its values. The second multiply-inherited class specifies the member parameter type. We call this a multi-value parameter. Multi-value parameters have additional constraints (min/max member count). Note that a defaultValue for a multi-value parameter is a list. Thus, multi-value parameters have a new field, the `itemDefaultValue` key, which is the default value of a new list member when unspecified. This allows the user to simply specify min/max/itemDefaultValue *in lieu* of the defaultValue, and an appropriate defaultValue of length min will be created by cloning the itemDefaultValue min times. Given the "wrapping" nature of this class, it overrides every method in the Parameter class except for validate, because it effectively overrides all value parsing/checking and calls the second-level multiply-inherited class's parse/check methods for member values. """ def __init__(self,min=None,max=None,itemDefaultValue=None, multiValueTitle=None): self.multiValue = True self.min = min self.max = max self.multiValueTitle = multiValueTitle self._itemDefaultValue = itemDefaultValue self.setItemDefaultValue(itemDefaultValue) # Fill in self.defaultValue from itemDefaultValue iff # len(self.defaultValue) == 0, and if there's an item default value. if self.defaultValue is not None \ and not isinstance(self.defaultValue,list): raise PortalError("invalid multivalue parameter '%s' defaultValue (%s):" " not a list" % (str(self.defaultValue),self.name)) if self.min is not None and self.min > 0 \ and (self.defaultValue is None or self.defaultValue == []): # or len(self.defaultValue) < self.min): if self.itemDefaultValue is None: raise PortalError( "invalid multivalue parameter '%s' with min=%d:" " 0-length defaultValue (%s) and no itemdefaultValue specified!" % (self.name,self.min,str(self.defaultValue))) else: self._defaultValue = [ self.itemDefaultValue for x in range(0,self.min) ] @property def itemDefaultValue(self): """ Returns the new item default value. If a frontend can dynamically add member values to this Multi value parameter, this should be the value it autofills for the new member. """ return self._itemDefaultValue def setItemDefaultValue(self,value): LOG.debug("%s(%s)" % (self.name,str(value))) v = super(Multi,self)._parseValue(value) super(Multi,self)._checkValue(v) self._itemDefaultValue = v LOG.debug("%s(%s) -> %s" % (self.name,str(value),str(self._itemDefaultValue))) def _checkValue(self,value): if value is None and (self.min == None or self.min == 0): return if not isinstance(value,list): raise IllegalParameterValueError(value,param=self) for x in value: LOG.debug("%s(%s)" % (str(super(Multi,self)._checkValue),str(x))) super(Multi,self)._checkValue(x) def _parseValue(self,value): LOG.debug("%s(%s)" % (str(self.name),str(value))) if value == None: value = [] elif isinstance(value,six.string_types): if value == "": value = [] else: try: value = json.loads(value) except: raise PortalError( "invalid multivalue parameter JSON string (%s=%s)" % (self.name,str(value))) if not isinstance(value,list): raise PortalError("invalid multivalue parameter JSON value ('%s'): not list" % (str(value),)) for x in value: LOG.debug("x = %s" % (str(x))) nvalue = [ super(Multi,self)._parseValue(x) for x in value ] LOG.debug("%s(%s) -> %s" % (str(self.name),str(value),str(nvalue))) return nvalue def setValue(self,value): LOG.debug("%s(%s)" % (str(self.name),str(value))) self._checkValue(value) self._value = value return self.value def setDefaultValue(self,value): LOG.debug("%s(%s)" % (self.name,str(value))) if self.min is not None and self.min > 0 \ and (value is None or len(value) < self.min): raise PortalError( "invalid multivalue parameter '%s' with min=%d:" " insufficient default value (%s) specified!" % (self.name,self.min,str(value))) if value == None: self._defaultValue = [] return newValue = [] for v in value: LOG.debug("%s(%s)" % (self.name,str(v))) v = super(Multi,self)._parseValue(v) super(Multi,self)._checkValue(v) newValue.append(v) self._defaultValue = newValue LOG.debug("%s -> %s" % (self.name,str(newValue))) def validate(self): LOG.debug(self.name) self.setItemDefaultValue(self.itemDefaultValue) self.setDefaultValue(self.defaultValue) #self._parseValue(self.defaultValue) #self._checkValue(self.defaultValue) if self.legalValues: for v in self.legalValues: super(Multi,self)._parseValue(v) def toParamdef(self): d = super(Multi,self).toParamdef() fields = [ "multiValue","min","max","itemDefaultValue","multiValueTitle" ] for f in fields: d[f] = getattr(self,f) return d
[docs]class MultiParameter(Multi,Parameter): def __init__(self,name,description,type,defaultValue,legalValues=None, longDescription=None,groupId=None,hide=False, prefix="emulab.net.parameter.",inputFieldHint=None, inputConstraints=None, min=None,max=None,itemDefaultValue=None,multiValueTitle=None): Parameter.__init__( self,name,description,type,defaultValue,legalValues=legalValues, longDescription=longDescription,groupId=groupId,hide=hide, prefix=prefix,inputFieldHint=inputFieldHint, inputConstraints=inputConstraints,_skipInitialChecks=True) Multi.__init__( self,min=min,max=max,itemDefaultValue=itemDefaultValue, multiValueTitle=multiValueTitle) # NB: we _skipInitialChecks because of interactions between the # Multi and Parameter methods, so both parent constructors must be # called before we validate. However, we need not wait to check # default/legal/itemDefault values. self.validate()
# def validate(self): # Parameter.validate(self) # self.setItemDefaultValue(self.itemDefaultValue) # self.setDefaultValue(self.defaultValue)
[docs]class StructParameter(Parameter): def __init__(self,name,description,defaultValue=None,members=[], longDescription=None,groupId=None,hide=False, prefix="emulab.net.parameter.", inputConstraints=None,_skipInitialChecks=False): self.parameters = {} self.parameterOrder = [] for m in members: self.addParameter(m) super(StructParameter,self).__init__( name,description,ParameterType.STRUCT,defaultValue, longDescription=longDescription,groupId=groupId,hide=hide,prefix=prefix, inputConstraints=inputConstraints,_skipInitialChecks=_skipInitialChecks) def addParameter(self,p): self.parameterOrder.append(p.name) self.parameters[p.name] = p @property def defaultValue(self): if self._defaultValue is not None: return self._defaultValue v = {} for x in self.parameterOrder: v[x] = self.parameters[x].defaultValue return v @property def legalValues(self): """ Struct parameters do not have legal values; those must be set on each member parameter. """ return None def _checkValue(self,value): if not isinstance(value,dict): raise PortalError("invalid struct parameter '%s' value '%s': not a dict" % (self.name,str(value))) for x in sorted(value.keys()): if not x in self.parameters: raise MissingParameterMemberError(self,x) self.parameters[x]._checkValue(value[x]) def _parseValue(self,value): LOG.debug("%s(%s)" % (self.name,str(value))) if value == None or value == "": value = {} elif isinstance(value,six.string_types): try: value = json.loads(value) except: raise PortalError("invalid struct parameter JSON string") if not isinstance(value,dict): raise PortalError("invalid struct parameter '%s' value: not dict (%s)" % (self.name,str(value))) nvalue = {} # Process supplied parameters (and error on anything extra). for x in sorted(value.keys()): if not x in self.parameters: raise PortalError("unknown struct member '%s' in value" % (x)) nvalue[x] = self.parameters[x]._parseValue(value[x]) # Process default values for child params that were not supplied. for x in sorted(self.parameters.keys()): if x in nvalue: continue nvalue[x] = self.parameters[x]._parseValue(self.parameters[x].defaultValue) for x in self.parameters: if not x in value.keys(): nvalue[x] = self.parameters[x].defaultValue LOG.debug("%s(%s) -> %s" % (self.name,str(value),str(nvalue))) return DictNamespace(nvalue)
[docs] def setValue(self,value): self._checkValue(value) for x in sorted(value.keys()): self.parameters[x].setValue(value[x]) self._value = value return self.value
[docs] def toParamdef(self): fields = [ "name","description","longDescription","type","_defaultValue", "groupId","hide","parameterOrder","inputConstraints" ] d = dict() for f in fields: fname = f if fname.startswith('_'): fname = fname[1:] d[fname] = getattr(self,f) d["parameters"] = {} for name in self.parameters: d["parameters"][name] = self.parameters[name].toParamdef() return d
[docs] def validate(self): for x in self.parameterOrder: self.parameters[x].validate()
[docs]class MultiStructParameter(Multi,StructParameter): def __init__(self,name,description,defaultValue=None,members=[], longDescription=None,groupId=None,hide=False, min=None,max=None,itemDefaultValue=None, multiValueTitle=None, prefix="emulab.net.parameter.",inputConstraints=None): StructParameter.__init__( self,name,description,defaultValue,members=members, longDescription=longDescription,groupId=groupId,hide=hide,prefix=prefix, inputConstraints=inputConstraints,_skipInitialChecks=True) Multi.__init__( self,min=min,max=max,itemDefaultValue=itemDefaultValue, multiValueTitle=multiValueTitle) self.validate()
[docs] def validate(self): for x in self.parameterOrder: self.parameters[x].validate() self._checkValue(self.defaultValue) self._parseValue(self.defaultValue)
[docs]class Context (object): """Handle context for scripts being run inside a portal. This class handles context for the portal, including where to put output RSpecs and handling parameterized scripts. Scripts using this class can also be run "standalone" (ie. not by the portal), in which case they take parameters on the command line and put RSpecs on the standard output. This class is a singleton. Most programs should access it through the portal.context variable; any additional "instances" of the object will be references to this.""" """This is a singleton class; only one can exist at a time This is implemented by overriding __new__""" _instance = None _initialized = False def __new__(cls, *args, **kwargs): if not cls._instance: cls._instance = super(Context, cls).__new__(cls, *args, **kwargs) return cls._instance def __init__ (self): # If someone accidentally calls the constructor on the singleton, this # prevents us for wiping out its previous state if self.__class__._initialized: return self._request = None self._suppressAutoPrint = False self._parameters = {} self._parameterGroups = { "advanced": "Advanced" } self._parameterOrder = [] self._parameterErrors = [] self._parameterWarnings = [] self._parameterWarningsAreFatal = False self._bindingDone = False self._envParams = {} if 'GENILIB_PORTAL_MODE' in os.environ: self._standalone = False self._portalRequestPath = os.environ.get('GENILIB_PORTAL_REQUEST_PATH',None) self._dumpParamsPath = os.environ.get('GENILIB_PORTAL_DUMPPARAMS_PATH',None) self._readParamsPath = os.environ.get('GENILIB_PORTAL_PARAMS_PATH',None) self._parameterWarningsAreFatal = \ bool(os.environ.get('GENILIB_PORTAL_WARNINGS_ARE_FATAL',None)) else: self._standalone = True self._portalRequestPath = None self.__class__._initialized = True
[docs] def bindRequestRSpec (self, rspec): """Bind the given request RSpec to the context, so that it can be automatically used with methods like printRequestRSpec. At the present time, only one request can be bound to a context""" if self._request is None: self._request = rspec # This feature removed until we can think through all the corner cases # better #sys.excepthook = self._make_excepthook() #atexit.register(self._autoPrintRequest) else: raise MultipleRSpecError
[docs] def makeRequestRSpec (self): """Make a new request RSpec, bind it to this context, and return it""" rspec = Request() self.bindRequestRSpec(rspec) return rspec
[docs] def printRequestRSpec (self, rspec = None): """Print the given request RSpec, or the one bound to this context if none is given. If run standalone (not in the portal), the request will be printed to the standard output; if run in the portal, it will be placed someplace the portal can pick it up. If the given rspec does not have a Tour object, this will attempt to build one from the file's docstring""" if rspec is None: if self._request is not None: rspec = self._request else: raise NoRSpecError("None supplied or bound to context") if not rspec.hasTour(): tour = igext.Tour() if tour.useDocstring(): rspec.addTour(tour) if any(self._parameters): rspec.ParameterData(self._parameters) self._suppressAutoPrint = True rspec.writeXML(self._portalRequestPath)
[docs] def defineParameter (self, name, description, typ, defaultValue, legalValues = None, longDescription = None, inputFieldHint = None, inputConstraints = None, advanced = False, groupId = None, hide=False, multiValue=False,min=None,max=None,itemDefaultValue=None,multiValueTitle=None, prefix="emulab.net.parameter."): """Define a new paramter to the script. The given name will be used when parameters are bound. The description is brief help text that will be shown to the user when making his/her selection. The type should be one of the types defined by ParameterType. defaultValue is required, but legalValues (a list) is optional; the defaultValue must be one of the legalValues. Entries in the legalValues list may be either simple strings (eg. "m400"), in which case they will be show directly to the user, or 2-element tuples (eg. ("m400", "ARM64"),), in which the second entry is what is shown to the user. defaultValue may be a tuple, so that one can pass, say, 'legalvalues[0]' for the option. The longDescription is an optional, detailed description of this parameter and how it relates to other parameters; it will be shown to the user if they ask to see the help, or as a pop-up/tooltip. advanced, group, and groupName all provide parameter group abstractions. Parameter groups are hidden by default from the user, and the user can expand them to view and modify them if desired. By setting advanced to True, you create a parameter group named "Advanced Parameters"; this group will not exist or be shown if none of your parameters set the 'advanced' argument to True. After defining parameters, bindParameters() must be called exactly once.""" if isinstance(defaultValue, tuple): defaultValue = defaultValue[0] # Backwards compat for the advanced key. if advanced and groupId == None: groupId="advanced" if multiValue: p = MultiParameter( name,description,typ,defaultValue,legalValues=legalValues, longDescription=longDescription,groupId=groupId,hide=hide, min=min,max=max,prefix=prefix,inputFieldHint=inputFieldHint, itemDefaultValue=itemDefaultValue,multiValueTitle=multiValueTitle, inputConstraints=inputConstraints) else: p = Parameter( name,description,typ,defaultValue,legalValues=legalValues, longDescription=longDescription,groupId=groupId,hide=hide, prefix=prefix,inputFieldHint=inputFieldHint, inputConstraints=inputConstraints) self.addParameter(p) return p
def defineStructParameter(self,name,description,defaultValue=None, longDescription=None,advanced=False,hide=False, members=[],multiValue=False,min=None,max=None, itemDefaultValue=None,multiValueTitle=None, inputConstraints=None, prefix="emulab.net.parameter."): if multiValue: p = MultiStructParameter( name,description,defaultValue,longDescription=longDescription, hide=hide,min=min,max=max,itemDefaultValue=itemDefaultValue, multiValueTitle=multiValueTitle,prefix=prefix,members=members, inputConstraints=inputConstraints) else: p = StructParameter( name,description,defaultValue,longDescription=longDescription, hide=hide,prefix=prefix,members=members) self.addParameter(p) return p def addParameter(self,parameter): self._parameterOrder.append(parameter.name) self._parameters[parameter.name] = parameter if len(self._parameters) == 1: atexit.register(self._checkBind)
[docs] def defineParameterGroup(self, groupId, groupName): """ Define a parameter group. Parameters may be added to this group, which has an identifying token composed of alphanumeric characters (groupId), and a human-readable name (groupName). Groups are intended to be used for advanced parameters; in the portal UI, they hidden in an expandable panel with the groupName --- and the user can choose to see and modify them, or not. You do not need to specify any groups; you can simply stuff all your parameters into the "Advanced Parameters" group by setting the 'advanced' argument of defineParameter to True. If you need multiple groups, define your own groups this way. """ self._parameterGroups[groupId] = groupName
[docs] def bindParameters (self,altParamSrc=None): """Returns values for the parameters defined by defineParameter(). Returns a Namespace (like argparse), so if you call foo = bindParameters(), a parameter defined with name "bar" is accessed as foo.bar . Since defaults are required, all parameters are guaranteed to have values in the Namespace If run standaline (not in the portal), parameters are pulled from the command line (try running with --help); if run in the portal, they are pulled from the portal itself. Or, if you provide the altParamSrc argument, you can specify your own parameters. If altParamSrc is a dict, we will bind the params as a dict, using the keys as parameter names, and the values as parameter values. If altParamSrc is a geni.rspec.pgmanifest.Manifest, we will extract the parameters and their values from the Manifest. Finally, if altParamSrc is a string, we'll try to parse it as a PG manifest xml document. No other forms of altParamSrc are currently specified.""" for paramName in self._parameterOrder: self._parameters[paramName].validate() self._bindingDone = True if altParamSrc: if isinstance(altParamSrc, dict): return self._bindParametersDict(altParamSrc) elif isinstance(altParamSrc, pgmanifest.Manifest): return self._bindParametersManifest(altParamSrc) elif isinstance(altParamSrc, (six.string_types)): try: manifestObj = pgmanifest.Manifest(xml=altParamSrc) return self._bindParametersManifest(manifestObj) except: ex = sys.exc_info()[0] raise ParameterBindError("assumed str altParamSrc was xml manifest, but" " parse error: %s" % (str(ex),)) else: raise ParameterBindError("unknown altParamSrc type: %s" % (str(type(altParamSrc)),)) elif self._standalone: return self._bindParametersCmdline() else: if self._dumpParamsPath: self._dumpParamsJSON() return self._bindParametersEnv()
[docs] def makeParameterWarningsFatal (self): """ Enable this option if you want to return an error to the user for incorrect parameter values, even if they can be autocorrected. This can be useful to show the user that """ self._parameterWarningsAreFatal = True
[docs] def reportError (self,parameterError,immediate=False): """ Report a parameter error to the portal. @parameterError is an exception object of type ParameterError. If @immediate is True, your script will exit immediately at this point with a dump of the errors (and fatal warnings, if enabled via Context.makeParameterWarningsFatal) in JSON format. If @immediate is False, the errors will accumulate until Context.verifyParameters is called (and the errors will then be printed). """ self._parameterErrors.append(parameterError) if immediate: self.verifyParameters()
[docs] def reportWarning (self,parameterError): """ Record a parameter warning. Warnings will be printed if there are other errors or if warnings have been set to be fatal, when Context.verifyParameters() is called, or when there is another subsequent immediate error. """ self._parameterWarnings.append(parameterError)
def _splitParamPathIntoComponents(self,paramPath): s1 = paramPath.split('.') s2 = [] for c in s1: sidx = c.find('[') if sidx > -1: try: idx = int(c[sidx+1:-1]) except: raise PortalError("invalid parameter path '%s': malformed index in component '%s'" % (paramPath,c)) s2.append(c[:sidx]) s2.append(idx) else: s2.append(c) return s2 def _getEnvParamForPath(self,paramPath): if self._standalone: raise PortalError("not in portal mode; cannot call _getEnvParamForPath") if isinstance(paramPath,six.string_types): paramPath = self._splitParamPathIntoComponents(paramPath) # Try to find the original value dict specified by the path, # probably because we want to annotate it: v = self._envParams["bindings"] lv = None for comp in paramPath: try: lv = v[comp] LOG.debug("lv = %s" % (str(lv))) v = lv["value"] LOG.debug("v = %s" % (str(v))) except: if dodebug: import traceback traceback.print_exc() raise PortalError( "nonexistent parameter value at component '%s' in path '%s'" % (str(comp),str(paramPath))) return lv
[docs] def verifyParameters (self): """ If there have been calls to Context.parameterError, and/or to Context.parameterWarning (and Context.makeParameterWarningsFatal has been called, making warnings fatal), this function will spit out some nice JSON-formatted exception info on stderr """ if len(self._parameterErrors) == 0 \ and (len(self._parameterWarnings) == 0 \ or not self._parameterWarningsAreFatal): return 0 if not self._standalone and self._readParamsPath is not None: # # Return the same blob to the frontend that we received, but # annotate it with errors/warnings and changed values: # erridx = 1 if len(self._parameterErrors): self._envParams["errors"] = {} for err in self._parameterErrors: self._envParams["errors"][str(erridx)] = {} for param in err.params: try: v = self._getEnvParamForPath(param) LOG.debug("v = %s" % (str(v))) except Exception as e: newmsg = "Double fault: while trying to generate error (%s, %s), encountered malformed parameter path: %s" % (str(param),err.message,e.message) self._envParams["errors"][str(erridx)] = dict(message=newmsg) erridx += 1 continue else: if not "errors" in v: v["errors"] = [] v["errors"].append(str(erridx)) for param in err.fixedValues.keys(): try: v = self._getEnvParamForPath(param) LOG.debug("v = %s" % (str(v))) except Exception as e: newmsg = "Double fault: while trying to update value (%s, %s), encountered malformed parameter path: %s" % (str(param),err.message,e.message) self._envParams["errors"][str(erridx)] = dict(message=newmsg) erridx += 1 continue else: v["fixedValue"] = err.fixedValues[param] self._envParams["errors"][str(erridx)] = dict(message=err.message) erridx += 1 if len(self._parameterWarnings): self._envParams["warnings"] = {} for err in self._parameterWarnings: self._envParams["warnings"][str(erridx)] = {} for param in err.params: try: v = self._getEnvParamForPath(param) except Exception as e: newmsg = "Double fault: while trying to generate warning (%s, %s), encountered malformed parameter path: %s" % (str(param),err.message,e.message) self._envParams["warnings"][str(erridx)] = dict(message=newmsg) erridx += 1 continue else: if not "warnings" in v: v["warnings"] = [] v["warnings"].append(str(erridx)) for param in err.fixedValues.keys(): try: v = self._getEnvParamForPath(param) LOG.debug("v = %s" % (str(v))) except Exception as e: newmsg = "Double fault: while trying to update value (%s, %s), encountered malformed parameter path: %s" % (str(param),err.message,e.message) self._envParams["errors"][str(erridx)] = dict(message=newmsg) erridx += 1 continue else: v["fixedValue"] = err.fixedValues[param] self._envParams["warnings"][str(erridx)] = dict(message=err.message) erridx += 1 json.dump(self._envParams,sys.stderr,cls=PortalJSONEncoder) else: # # Dump a JSON list of typed errors. # ea = [] ea.extend(self._parameterErrors) ea.extend(self._parameterWarnings) json.dump(ea,sys.stderr,cls=PortalJSONEncoder) # # Exit with a count of errors and (fatal) warnings, added to 100 ... # try to distinguish ourselves meaningfully! # retcode = len(self._parameterErrors) if self._parameterWarningsAreFatal: retcode += len(self._parameterWarnings) sys.exit(100+retcode)
[docs] def suppressAutoPrint (self): """ Suppress the automatic printing of the bound RSpec that normally happens when the program exits. """ self._suppressAutoPrint = True
def _bindParametersCmdline (self): parser = argparse.ArgumentParser() for name in self._parameterOrder: p = self._parameters[name] LOG.debug("%s = %s" % (str(p.name),str(p.defaultValue))) # Brutal hack to force p._parseValue to be called. Argparse will # only invoke the `type` function for the unsupplied default case # if the value is any type of string. default = p.defaultValue if isinstance(default,dict) or isinstance(default,list): default = json.dumps(default,sort_keys=True, separators=(',',':')) parser.add_argument("--" + name, type = p._parseValue, default = default, choices = p.legalValues, help = p.description) args = parser.parse_args() for name in self._parameterOrder: self._parameters[name].setValue(getattr(args, name)) return DictNamespace(args.__dict__) def _flattenEnvParams(self,p): """ The parameter block we get back from the frontend may be a giant dict with metadata and values specified as "value": <value> pairs within each parameter descriptor. So "flatten" the value pairs into the right data structure. """ if not isinstance(p,list) and not isinstance(p,dict): return p elif isinstance(p,dict): if "value" in p: return self._flattenEnvParams(p["value"]) ret = {} for k in p.keys(): ret[k] = self._flattenEnvParams(p[k]) LOG.debug("ret -> %s" % (str(ret))) elif isinstance(p,list): ret = [] for x in p: ret.append(self._flattenEnvParams(x)) LOG.debug("ret -> %s" % (str(ret))) return ret def _bindParametersEnv (self): """ Read parameter values from the environment (e.g., from the frontend). You should only use this path if you understand the representation(s) the frontend uses to send parameter values. """ if self._readParamsPath: f = open(self._readParamsPath, "r") self._envParams = json.load(f) f.close() if len(self._envParams): self._flattenedEnvParams = self._flattenEnvParams(self._envParams["bindings"]) else: self._flattenedEnvParams = {} LOG.debug("flattened: %s" % (str(self._flattenedEnvParams))) return self._bindParametersDict(self._flattenedEnvParams) def _bindParametersDict(self,paramValues): """ This is the generic parameter value parse/bind function. For each defined parameter (self._parameters.keys()), in definition order (self._parameterOrder), extract a value from the supplied value. Each Parameter subclass provides a _parseValue method to perform the extraction, and can thus define the range of input values it supports. Once a value is extracted, we set that value (via the setValue method) on the unbound parameter, so that self._parameters[name].value contains the extracted value. Note that setValue may throw an error even if _parseValue did not, since there may be additional constraints other than type correctness (e.g., a range of allowed values.) """ namespace = DictNamespace() for name in self._parameterOrder: p = self._parameters[name] val = paramValues.get(name, p.defaultValue) LOG.debug("paramValue(%s): %s" % (str(p),str(val))) try: val = p._parseValue(val) except ParameterError as e: if dodebug: import traceback traceback.print_exc() self.reportError(e) continue except Exception as e: if dodebug: import traceback traceback.print_exc() self.reportError( ParameterError("Could not parse '%s' value '%s': %s" % (name,str(val),str(e)),[name])) continue try: p.setValue(val) except ParameterError as e: if dodebug: import traceback traceback.print_exc() self.reportError(e) continue except Exception as e: if dodebug: import traceback traceback.print_exc() self.reportError( ParameterError("Illegal value '%s' for parameter '%s': %s" % (str(val),name,str(e)),[name])) continue setattr(namespace, name, val) # This might not return. self.verifyParameters() self._bindingDone = True return namespace def _bindParametersManifest(self,manifest): """ This method extracts parameter values from a `geni.rspec.pgmanifest.Manifest` class, places them in a dict, and invokes self._bindParametersDict. """ pdict = {} for manifestParameter in manifest.parameters: pdict[manifestParameter.name] = manifestParameter.value return self._bindParametersDict(pdict) def _dumpParamsJSON (self): # # Output the parameter dict in sorted order (sorted in terms of parameter # definition order). This is correct, identical to json.dump (other than # key order), and much easier than subclassing json.JSONEncoder :). # didFirst = False f = open(self._dumpParamsPath, "w+") f.write('{') for name in self._parameterOrder: if didFirst: f.write(', ') else: didFirst = True p = self._parameters[name] pd = p.toParamdef() # A little backwards-compat hack. if p.groupId and p.groupId in self._parameterGroups: pd['groupName'] = self._parameterGroups[p.groupId] json.dump(name,f) f.write(': ') json.dump(pd,f) f.write('}') f.close() return def _checkBind (self): if len(self._parameters) > 0 and not self._bindingDone: warnings.warn("Parameters were defined, but never bound with " + " bindParameters()", RuntimeWarning) def _autoPrintRequest (self): if not self._suppressAutoPrint: self.printRequestRSpec() def _make_excepthook (self): old_excepthook = sys.excepthook def _excepthook(type, value, traceback): self.suppressAutoPrint() return old_excepthook(type, value, traceback) return _excepthook
context = Context() """ Module-global Context object - most users of this module should simply use this rather than trying to create a new Context object """ def get_context(): return context
[docs]class PortalJSONEncoder(json.JSONEncoder):
[docs] def default(self, o): if isinstance(o,PortalError): return o.__objdict__() else: # First try the default encoder: try: return json.JSONEncoder.default(self, o) except Exception: try: # Then try to return a string, at least return str(o) except Exception: # Let the base class default method raise the TypeError return json.JSONEncoder.default(self, o)
# # Define some exceptions. Everybody should subclass PortalError. #
[docs]class PortalError (Exception): def __init__(self,message): super(PortalError, self).__init__() self.message = message def __str__(self): return self.__class__.__name__ + ": " + self.message def __objdict__(self): retval = dict({ 'errorType': self.__class__.__name__, }) for k in self.__dict__.keys(): if k == 'errorType': continue if k.startswith('_'): continue retval[k] = self.__dict__[k] return retval
[docs]class ParameterError (PortalError): """ A simple class to describe a parameter error. If you need to report an error with a user-specified parameter value to the Portal UI, please create (don't throw) one of these error objects, and tell the Portal about it by calling Context.reportError. """ def __init__(self,message,paramList,fixedValues={}): """ Create a ParameterError. @message is the overall error message; in the Portal Web UI, it will be displayed near each involved parameter for maximal impact. @paramList is a list of the parameters that are involved in the error (often it is the combination of parameters that creates the error condition). The Portal UI will show this error message near *each* involved parameter to increase user understanding of the error. """ super(ParameterError, self).__init__(message) self.params = paramList if not fixedValues: fixedValues = {} self.fixedValues = fixedValues
[docs]class ParameterWarning (PortalError): """ A simple class to describe a parameter warning. If you need to report an warning with a user-specified parameter value to the Portal UI, please create (don't throw) one of these error objects, and tell the Portal about it by calling Context.reportWarning . The first time the Portal UI runs your geni-lib script with a user's parameter values, it turns on the "warnings are fatal" mode (and then warnings are reported as errors). This gives you a chance to warn the user that they might be about to do something stupid, and/or suggest a set of modified values that will improve the situation. . """ def __init__(self,message,paramList,fixedValues={}): """ Create a ParameterWarning. @message is the overall error message; in the Portal Web UI, it will be displayed near each involved parameter for maximal impact. @paramList is a list of the parameters that are involved in the warning (often it is the combination of parameters that creates the error condition). The Portal UI will show this warning message near *each* involved parameter to increase user understanding of the error. If you supply the @fixedValue dict, the Portal UI will change the values the user submitted to those you suggest (and it will tell them it did so). You might want to supply @fixedValues for a proper warning, because if something is only a warning, that implies your script can and will proceed in the absence of further user input. But sometimes we want to let the user know that a parameter change will occur, so we warn them and autocorrect! """ super(ParameterWarning, self).__init__(message) self.params = paramList if not fixedValues: fixedValues = {} self.fixedValues = fixedValues
[docs]class MissingParameterMemberError(PortalError): def __init__(self,param,memberName): super(MissingParameterMemberError, self).__init__( memberName + " not in " + param.name) self._param = param
[docs]class IllegalParameterDefaultError (PortalError): def __init__ (self,val,param=None): super(IllegalParameterDefaultError, self).__init__("no message?") self._val = val self._param = param def __str__ (self): namestr = "" if self._param: namestr = " for parameter '%s'" % (str(self._param.name)) return "%s given as a default value%s, but is not listed as a legal value" % (str(self._val),namestr)
[docs]class IllegalParameterValueError (PortalError): def __init__ (self,val,param=None): super(IllegalParameterValueError, self).__init__("no message?") self._val = val self._param = param def __str__ (self): namestr = "" if self._param: namestr = " for parameter '%s'" % (str(self._param.name)) return "value '%s'%s is not a legal value" % (str(self._val),namestr)
[docs]class ParameterBindError (PortalError): def __init__ (self,val,param=None): super(ParameterBindError, self).__init__("no message?") self._val = val self._param = param def __str__ (self): namestr = "" if self._param: namestr = " for parameter '%s'" % (str(self._param.name)) return "bad parameter binding: %s%s" % (str(self._val),namestr)
[docs]class NoRSpecError (PortalError): def __init__ (self,val): super(NoRSpecError, self).__init__("no message?") self._val = val def __str__ (self): return "No RSpec given: %s" % str(self._val,)
[docs]class MultipleRSpecError (PortalError): def __init__ (self,val): super(MultipleRSpecError, self).__init__("no message?") self._val = val def __str__ (self): return "Only one RSpec can be bound to a portal.Context"