1010 lines
44 KiB
Python
1010 lines
44 KiB
Python
# This code is part of Ansible, but is an independent component.
|
|
# This particular file snippet, and this file snippet only, is BSD licensed.
|
|
# Modules you write using this snippet, which is embedded dynamically by Ansible
|
|
# still belong to the author of the module, and may assign their own license
|
|
# to the complete work.
|
|
#
|
|
# Copyright (c) 2017, Sumit Kumar <sumit4@netapp.com>
|
|
# Copyright (c) 2017, Michael Price <michael.price@netapp.com>
|
|
# All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without modification,
|
|
# are permitted provided that the following conditions are met:
|
|
#
|
|
# * Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# * Redistributions in binary form must reproduce the above copyright notice,
|
|
# this list of conditions and the following disclaimer in the documentation
|
|
# and/or other materials provided with the distribution.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
|
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
|
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
|
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
'''
|
|
netapp.py
|
|
'''
|
|
|
|
from __future__ import (absolute_import, division, print_function)
|
|
__metaclass__ = type
|
|
|
|
import base64
|
|
import logging
|
|
import os
|
|
import ssl
|
|
import time
|
|
from ansible.module_utils.basic import missing_required_lib
|
|
from ansible.module_utils._text import to_native
|
|
|
|
try:
|
|
from ansible.module_utils.ansible_release import __version__ as ansible_version
|
|
except ImportError:
|
|
ansible_version = 'unknown'
|
|
|
|
COLLECTION_VERSION = "21.18.1"
|
|
CLIENT_APP_VERSION = "%s/%s" % ("%s", COLLECTION_VERSION)
|
|
IMPORT_EXCEPTION = None
|
|
|
|
try:
|
|
from netapp_lib.api.zapi import zapi
|
|
HAS_NETAPP_LIB = True
|
|
except ImportError as exc:
|
|
HAS_NETAPP_LIB = False
|
|
IMPORT_EXCEPTION = exc
|
|
|
|
try:
|
|
import requests
|
|
HAS_REQUESTS = True
|
|
except ImportError:
|
|
HAS_REQUESTS = False
|
|
|
|
HAS_SF_SDK = False
|
|
SF_BYTE_MAP = dict(
|
|
# Management GUI displays 1024 ** 3 as 1.1 GB, thus use 1000.
|
|
bytes=1,
|
|
b=1,
|
|
kb=1000,
|
|
mb=1000 ** 2,
|
|
gb=1000 ** 3,
|
|
tb=1000 ** 4,
|
|
pb=1000 ** 5,
|
|
eb=1000 ** 6,
|
|
zb=1000 ** 7,
|
|
yb=1000 ** 8
|
|
)
|
|
|
|
POW2_BYTE_MAP = dict(
|
|
# Here, 1 kb = 1024
|
|
bytes=1,
|
|
b=1,
|
|
k=1024,
|
|
m=1024 ** 2,
|
|
g=1024 ** 3,
|
|
t=1024 ** 4,
|
|
p=1024 ** 5,
|
|
e=1024 ** 6,
|
|
z=1024 ** 7,
|
|
y=1024 ** 8,
|
|
kb=1024,
|
|
mb=1024 ** 2,
|
|
gb=1024 ** 3,
|
|
tb=1024 ** 4,
|
|
pb=1024 ** 5,
|
|
eb=1024 ** 6,
|
|
zb=1024 ** 7,
|
|
yb=1024 ** 8,
|
|
)
|
|
|
|
ERROR_MSG = dict(
|
|
no_cserver='This module is expected to run as cluster admin'
|
|
)
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
LOG_FILE = '/tmp/ontap_apis.log'
|
|
|
|
try:
|
|
from solidfire.factory import ElementFactory
|
|
HAS_SF_SDK = True
|
|
except ImportError:
|
|
HAS_SF_SDK = False
|
|
|
|
|
|
def has_netapp_lib():
|
|
return HAS_NETAPP_LIB
|
|
|
|
|
|
def netapp_lib_is_required():
|
|
return "Error: the python NetApp-Lib module is required. Import error: %s" % str(IMPORT_EXCEPTION)
|
|
|
|
|
|
def has_sf_sdk():
|
|
return HAS_SF_SDK
|
|
|
|
|
|
def na_ontap_host_argument_spec():
|
|
|
|
return dict(
|
|
hostname=dict(required=True, type='str'),
|
|
username=dict(required=False, type='str', aliases=['user']),
|
|
password=dict(required=False, type='str', aliases=['pass'], no_log=True),
|
|
https=dict(required=False, type='bool', default=False),
|
|
validate_certs=dict(required=False, type='bool', default=True),
|
|
http_port=dict(required=False, type='int'),
|
|
ontapi=dict(required=False, type='int'),
|
|
use_rest=dict(required=False, type='str', default='auto'),
|
|
feature_flags=dict(required=False, type='dict', default=dict()),
|
|
cert_filepath=dict(required=False, type='str'),
|
|
key_filepath=dict(required=False, type='str', no_log=False),
|
|
)
|
|
|
|
|
|
def na_ontap_host_argument_spec_peer():
|
|
spec = na_ontap_host_argument_spec()
|
|
spec.pop('feature_flags')
|
|
# get rid of default values, as we'll use source values
|
|
for value in spec.values():
|
|
if 'default' in value:
|
|
value.pop('default')
|
|
return spec
|
|
|
|
|
|
def has_feature(module, feature_name):
|
|
feature = get_feature(module, feature_name)
|
|
if isinstance(feature, bool):
|
|
return feature
|
|
module.fail_json(msg="Error: expected bool type for feature flag: %s" % feature_name)
|
|
|
|
|
|
def get_feature(module, feature_name):
|
|
''' if the user has configured the feature, use it
|
|
otherwise, use our default
|
|
'''
|
|
default_flags = dict(
|
|
strict_json_check=True, # if true, fail if response.content in not empty and is not valid json
|
|
trace_apis=False, # if true, append ZAPI and REST requests/responses to /tmp/ontap_zapi.txt
|
|
check_required_params_for_none=True,
|
|
classic_basic_authorization=False, # use ZAPI wrapper to send Authorization header
|
|
deprecation_warning=True,
|
|
sanitize_xml=True,
|
|
sanitize_code_points=[8], # unicode values, 8 is backspace
|
|
show_modified=True,
|
|
always_wrap_zapi=True, # for better error reporting
|
|
flexcache_delete_return_timeout=5, # ONTAP bug if too big?
|
|
# for SVM, whch protocols can be allowed
|
|
svm_allowable_protocols_rest=['cifs', 'fcp', 'iscsi', 'nvme', 'nfs'],
|
|
svm_allowable_protocols_zapi=['cifs', 'fcp', 'iscsi', 'nvme', 'nfs', 'ndmp', 'http'],
|
|
max_files_change_threshold=1, # percentage of increase/decrease required to trigger a modify action
|
|
warn_or_fail_on_fabricpool_backend_change='fail',
|
|
no_cserver_ems=False # when True, don't attempt to find cserver and don't send cserver EMS
|
|
)
|
|
|
|
if module.params['feature_flags'] is not None and feature_name in module.params['feature_flags']:
|
|
return module.params['feature_flags'][feature_name]
|
|
if feature_name in default_flags:
|
|
return default_flags[feature_name]
|
|
module.fail_json(msg="Internal error: unexpected feature flag: %s" % feature_name)
|
|
|
|
|
|
def create_sf_connection(module, port=None):
|
|
hostname = module.params['hostname']
|
|
username = module.params['username']
|
|
password = module.params['password']
|
|
|
|
if HAS_SF_SDK and hostname and username and password:
|
|
try:
|
|
return ElementFactory.create(hostname, username, password, port=port)
|
|
except Exception as exc:
|
|
raise Exception("Unable to create SF connection: %s" % exc)
|
|
module.fail_json(msg="the python SolidFire SDK module is required")
|
|
|
|
|
|
def set_auth_method(module, username, password, cert_filepath, key_filepath):
|
|
error = None
|
|
if password is None and username is None:
|
|
if cert_filepath is None:
|
|
error = ('Error: cannot have a key file without a cert file' if key_filepath is not None
|
|
else 'Error: ONTAP module requires username/password or SSL certificate file(s)')
|
|
else:
|
|
auth_method = 'single_cert' if key_filepath is None else 'cert_key'
|
|
elif password is not None and username is not None:
|
|
if cert_filepath is not None or key_filepath is not None:
|
|
error = 'Error: cannot have both basic authentication (username/password) ' +\
|
|
'and certificate authentication (cert/key files)'
|
|
else:
|
|
auth_method = 'basic_auth' if has_feature(module, 'classic_basic_authorization') else 'speedy_basic_auth'
|
|
else:
|
|
error = 'Error: username and password have to be provided together'
|
|
if cert_filepath is not None or key_filepath is not None:
|
|
error += ' and cannot be used with cert or key files'
|
|
if error:
|
|
module.fail_json(msg=error)
|
|
return auth_method
|
|
|
|
|
|
def setup_host_options_from_module_params(host_options, module, keys):
|
|
'''if an option is not set, use primary value.
|
|
but don't mix up basic and certificate authentication methods
|
|
|
|
host_options is updated in place
|
|
option values are read from module.params
|
|
keys is a list of keys that need to be added/updated/left alone in host_options
|
|
'''
|
|
password_keys = ['username', 'password']
|
|
certificate_keys = ['cert_filepath', 'key_filepath']
|
|
use_password = any(host_options.get(x) is not None for x in password_keys)
|
|
use_certificate = any(host_options.get(x) is not None for x in certificate_keys)
|
|
if use_password and use_certificate:
|
|
module.fail_json(
|
|
msg='Error: host cannot have both basic authentication (username/password) and certificate authentication (cert/key files).')
|
|
if use_password:
|
|
exclude_keys = certificate_keys
|
|
elif use_certificate:
|
|
exclude_keys = password_keys
|
|
else:
|
|
exclude_keys = []
|
|
for key in keys:
|
|
if host_options.get(key) is None and key not in exclude_keys:
|
|
# use same value as source if no value is given for dest
|
|
host_options[key] = module.params[key]
|
|
|
|
|
|
def set_zapi_port_and_transport(server, https, port, validate_certs):
|
|
# default is HTTP
|
|
if https:
|
|
if port is None:
|
|
port = 443
|
|
transport_type = 'HTTPS'
|
|
# HACK to bypass certificate verification
|
|
if validate_certs is False and not os.environ.get('PYTHONHTTPSVERIFY', '') and getattr(ssl, '_create_unverified_context', None):
|
|
ssl._create_default_https_context = ssl._create_unverified_context
|
|
else:
|
|
if port is None:
|
|
port = 80
|
|
transport_type = 'HTTP'
|
|
server.set_transport_type(transport_type)
|
|
server.set_port(port)
|
|
|
|
|
|
def setup_na_ontap_zapi(module, vserver=None, wrap_zapi=False, host_options=None):
|
|
if host_options is None:
|
|
host_options = module.params
|
|
hostname = host_options.get('hostname')
|
|
username = host_options.get('username')
|
|
password = host_options.get('password')
|
|
cert_filepath = host_options.get('cert_filepath')
|
|
key_filepath = host_options.get('key_filepath')
|
|
https = host_options.get('https')
|
|
validate_certs = host_options.get('validate_certs')
|
|
port = host_options.get('http_port')
|
|
version = host_options.get('ontapi')
|
|
trace = has_feature(module, 'trace_apis')
|
|
if trace:
|
|
logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG, format='%(asctime)s %(levelname)-8s %(message)s')
|
|
wrap_zapi |= has_feature(module, 'always_wrap_zapi')
|
|
auth_method = set_auth_method(module, username, password, cert_filepath, key_filepath)
|
|
|
|
if not HAS_NETAPP_LIB:
|
|
module.fail_json(msg=netapp_lib_is_required())
|
|
|
|
# set up zapi
|
|
if auth_method in ('single_cert', 'cert_key'):
|
|
# override NaServer in netapp-lib to enable certificate authentication
|
|
server = OntapZAPICx(hostname, module=module, username=username, password=password,
|
|
validate_certs=validate_certs, cert_filepath=cert_filepath,
|
|
key_filepath=key_filepath, style=zapi.NaServer.STYLE_CERTIFICATE,
|
|
auth_method=auth_method, trace=trace)
|
|
# SSL certificate authentication requires SSL
|
|
https = True
|
|
elif auth_method == 'speedy_basic_auth' or wrap_zapi:
|
|
# override NaServer in netapp-lib to add Authorization header preemptively
|
|
# use wrapper to handle parse error (mostly for na_ontap_command)
|
|
server = OntapZAPICx(hostname, module=module, username=username, password=password,
|
|
validate_certs=validate_certs, auth_method=auth_method, trace=trace)
|
|
else:
|
|
# legacy netapp-lib
|
|
server = zapi.NaServer(hostname, username=username, password=password, trace=trace)
|
|
if vserver:
|
|
server.set_vserver(vserver)
|
|
if host_options.get('use_rest') == 'always':
|
|
note = '' if https else ' Note: https is set to false.'
|
|
module.warn("Using ZAPI for %s, ignoring 'use_rest: always'.%s" % (module._name, note))
|
|
|
|
set_zapi_port_and_transport(server, https, port, validate_certs)
|
|
server.set_api_version(major=1, minor=(version or 110))
|
|
server.set_server_type('FILER')
|
|
return server
|
|
|
|
|
|
def is_zapi_connection_error(message):
|
|
''' return True if it is a connection issue '''
|
|
# netapp-lib message may contain a tuple or a str!
|
|
try:
|
|
if isinstance(message, tuple) and isinstance(message[0], ConnectionError):
|
|
return True
|
|
except NameError:
|
|
# python 2.7 does not know about ConnectionError
|
|
pass
|
|
return isinstance(message, str) and message.startswith(('URLError', 'Unauthorized'))
|
|
|
|
|
|
def is_zapi_write_access_error(message):
|
|
''' return True if it is a write access error '''
|
|
# netapp-lib message may contain a tuple or a str!
|
|
if isinstance(message, str) and message.startswith('Insufficient privileges:'):
|
|
return 'does not have write access' in message
|
|
return False
|
|
|
|
|
|
def is_zapi_missing_vserver_error(message):
|
|
''' return True if it is a missing vserver error '''
|
|
# netapp-lib message may contain a tuple or a str!
|
|
return isinstance(message, str) and message == 'Vserver API missing vserver parameter.'
|
|
|
|
|
|
def ems_log_event_cserver(source, server, module):
|
|
if has_feature(module, 'no_cserver_ems'):
|
|
return
|
|
results = get_cserver(server)
|
|
cserver = setup_na_ontap_zapi(module=module, vserver=results)
|
|
ems_log_event(source, cserver)
|
|
|
|
|
|
def ems_log_event(source, server, name="Ansible", ident="12345", version=COLLECTION_VERSION,
|
|
category="Information", event="setup", autosupport="false"):
|
|
ems_log = zapi.NaElement('ems-autosupport-log')
|
|
# Host name invoking the API.
|
|
ems_log.add_new_child("computer-name", name)
|
|
# ID of event. A user defined event-id, range [0..2^32-2].
|
|
ems_log.add_new_child("event-id", ident)
|
|
# Name of the application invoking the API.
|
|
ems_log.add_new_child("event-source", source)
|
|
# Version of application invoking the API.
|
|
ems_log.add_new_child("app-version", version)
|
|
# Application defined category of the event.
|
|
ems_log.add_new_child("category", category)
|
|
# Description of event to log. An application defined message to log.
|
|
ems_log.add_new_child("event-description", event)
|
|
ems_log.add_new_child("log-level", "6")
|
|
ems_log.add_new_child("auto-support", autosupport)
|
|
try:
|
|
server.invoke_successfully(ems_log, True)
|
|
except zapi.NaApiError as exc:
|
|
# Do not fail if we can't connect to the server.
|
|
# The module will report a better error when trying to get some data from ONTAP.
|
|
# Do not fail if we don't have write privileges.
|
|
# Do not fail if there is no cserver, as on FSx.
|
|
if not is_zapi_connection_error(exc.message) \
|
|
and not is_zapi_write_access_error(exc.message) \
|
|
and not is_zapi_missing_vserver_error(exc.message):
|
|
# raise on other errors, as it may be a bug in calling the ZAPI
|
|
raise exc
|
|
|
|
|
|
def get_cserver_zapi(server):
|
|
''' returns None if not run on the management or cluster IP '''
|
|
vserver_info = zapi.NaElement('vserver-get-iter')
|
|
query_details = zapi.NaElement.create_node_with_children('vserver-info', **{'vserver-type': 'admin'})
|
|
query = zapi.NaElement('query')
|
|
query.add_child_elem(query_details)
|
|
vserver_info.add_child_elem(query)
|
|
try:
|
|
result = server.invoke_successfully(vserver_info,
|
|
enable_tunneling=False)
|
|
except zapi.NaApiError as exc:
|
|
# Do not fail if we can't connect to the server.
|
|
# The module will report a better error when trying to get some data from ONTAP.
|
|
if is_zapi_connection_error(exc.message):
|
|
return None
|
|
# raise on other errors, as it may be a bug in calling the ZAPI
|
|
raise exc
|
|
attribute_list = result.get_child_by_name('attributes-list')
|
|
if attribute_list is not None:
|
|
vserver_list = attribute_list.get_child_by_name('vserver-info')
|
|
if vserver_list is not None:
|
|
return vserver_list.get_child_content('vserver-name')
|
|
return None
|
|
|
|
|
|
def classify_zapi_exception(error):
|
|
''' return type of error '''
|
|
try:
|
|
# very unlikely to fail, but don't take any chance
|
|
err_code = int(error.code)
|
|
except (AttributeError, ValueError):
|
|
err_code = 0
|
|
try:
|
|
# very unlikely to fail, but don't take any chance
|
|
err_msg = error.message
|
|
except AttributeError:
|
|
err_msg = ""
|
|
if err_code == 13005 and err_msg.startswith('Unable to find API:') and 'data vserver' in err_msg:
|
|
return 'missing_vserver_api_error', 'Most likely running a cluster level API as vserver: %s' % to_native(error)
|
|
if err_code == 13001 and err_msg.startswith("RPC: Couldn't make connection"):
|
|
return 'rpc_error', to_native(error)
|
|
return "other_error", to_native(error)
|
|
|
|
|
|
def get_cserver(connection, is_rest=False):
|
|
if not is_rest:
|
|
return get_cserver_zapi(connection)
|
|
|
|
params = {'fields': 'type'}
|
|
api = "private/cli/vserver"
|
|
json, error = connection.get(api, params)
|
|
if json is None or error is not None:
|
|
# exit if there is an error or no data
|
|
return None
|
|
vservers = json.get('records')
|
|
if vservers is not None:
|
|
for vserver in vservers:
|
|
if vserver['type'] == 'admin': # cluster admin
|
|
return vserver['vserver']
|
|
if len(vservers) == 1: # assume vserver admin
|
|
return vservers[0]['vserver']
|
|
|
|
return None
|
|
|
|
|
|
if HAS_NETAPP_LIB:
|
|
class OntapZAPICx(zapi.NaServer):
|
|
''' override zapi NaServer class to:
|
|
- enable SSL certificate authentication
|
|
- ignore invalid XML characters in ONTAP output (when using CLI module)
|
|
- add Authorization header when using basic authentication
|
|
'''
|
|
def __init__(self, hostname=None, server_type=zapi.NaServer.SERVER_TYPE_FILER,
|
|
transport_type=zapi.NaServer.TRANSPORT_TYPE_HTTP,
|
|
style=zapi.NaServer.STYLE_LOGIN_PASSWORD, username=None,
|
|
password=None, port=None, trace=False, module=None,
|
|
cert_filepath=None, key_filepath=None, validate_certs=None,
|
|
auth_method=None):
|
|
# python 2.x syntax, but works for python 3 as well
|
|
super(OntapZAPICx, self).__init__(hostname, server_type=server_type,
|
|
transport_type=transport_type,
|
|
style=style, username=username,
|
|
password=password, port=port, trace=trace)
|
|
self.cert_filepath = cert_filepath
|
|
self.key_filepath = key_filepath
|
|
self.validate_certs = validate_certs
|
|
self.module = module
|
|
self.base64_creds = None
|
|
if auth_method == 'speedy_basic_auth':
|
|
auth = '%s:%s' % (username, password)
|
|
self.base64_creds = base64.b64encode(auth.encode()).decode()
|
|
|
|
def _create_certificate_auth_handler(self):
|
|
try:
|
|
context = ssl.create_default_context()
|
|
except AttributeError as exc:
|
|
self._fail_with_exc_info('SSL certificate authentication requires python 2.7 or later.', exc)
|
|
|
|
if not self.validate_certs:
|
|
context.check_hostname = False
|
|
context.verify_mode = ssl.CERT_NONE
|
|
try:
|
|
context.load_cert_chain(self.cert_filepath, keyfile=self.key_filepath)
|
|
except IOError as exc:
|
|
self._fail_with_exc_info('Cannot load SSL certificate, check files exist.', exc)
|
|
|
|
return zapi.urllib.request.HTTPSHandler(context=context)
|
|
|
|
def _fail_with_exc_info(self, arg0, exc):
|
|
msg = arg0
|
|
msg += ' More info: %s' % repr(exc)
|
|
self.module.fail_json(msg=msg)
|
|
|
|
def sanitize_xml(self, response):
|
|
# some ONTAP CLI commands return BEL on error
|
|
new_response = response.replace(b'\x07\n', b'')
|
|
# And 9.1 uses \r\n rather than \n !
|
|
new_response = new_response.replace(b'\x07\r\n', b'')
|
|
# And 9.7 may send backspaces
|
|
for code_point in get_feature(self.module, 'sanitize_code_points'):
|
|
if bytes([8]) == b'\x08': # python 3
|
|
byte = bytes([code_point])
|
|
elif chr(8) == b'\x08': # python 2
|
|
byte = chr(code_point)
|
|
else: # very unlikely, noop
|
|
byte = b'.'
|
|
new_response = new_response.replace(byte, b'.')
|
|
return new_response
|
|
|
|
def _parse_response(self, response):
|
|
''' handling XML parsing exception '''
|
|
try:
|
|
return super(OntapZAPICx, self)._parse_response(response)
|
|
except zapi.etree.XMLSyntaxError as exc:
|
|
if has_feature(self.module, 'sanitize_xml'):
|
|
try:
|
|
return super(OntapZAPICx, self)._parse_response(self.sanitize_xml(response))
|
|
except Exception:
|
|
# ignore a second exception, we'll report the first one
|
|
pass
|
|
try:
|
|
# report first exception, but include full response
|
|
exc.msg += ". Received: %s" % response
|
|
except Exception:
|
|
# in case the response is very badly formatted, ignore it
|
|
pass
|
|
raise exc
|
|
|
|
def _create_request(self, na_element, enable_tunneling=False):
|
|
''' intercept newly created request to add Authorization header '''
|
|
request, netapp_element = super(OntapZAPICx, self)._create_request(na_element, enable_tunneling=enable_tunneling)
|
|
request.add_header('X-Dot-Client-App', CLIENT_APP_VERSION % self.module._name)
|
|
if self.base64_creds is not None:
|
|
request.add_header('Authorization', 'Basic %s' % self.base64_creds)
|
|
return request, netapp_element
|
|
|
|
# as is from latest version of netapp-lib
|
|
def invoke_elem(self, na_element, enable_tunneling=False):
|
|
"""Invoke the API on the server."""
|
|
if not na_element or not isinstance(na_element, zapi.NaElement):
|
|
raise ValueError('NaElement must be supplied to invoke API')
|
|
|
|
request, request_element = self._create_request(na_element,
|
|
enable_tunneling)
|
|
|
|
if self._trace:
|
|
zapi.LOG.debug("Request: %s", request_element.to_string(pretty=True))
|
|
|
|
if not hasattr(self, '_opener') or not self._opener \
|
|
or self._refresh_conn:
|
|
self._build_opener()
|
|
try:
|
|
if hasattr(self, '_timeout'):
|
|
response = self._opener.open(request, timeout=self._timeout)
|
|
else:
|
|
response = self._opener.open(request)
|
|
except zapi.urllib.error.HTTPError as exc:
|
|
raise zapi.NaApiError(exc.code, exc.reason)
|
|
except zapi.urllib.error.URLError as exc:
|
|
msg = 'URL error'
|
|
error = repr(exc)
|
|
try:
|
|
# ConnectionRefusedError is not defined in python 2.7
|
|
if isinstance(exc.reason, ConnectionRefusedError):
|
|
msg = 'Unable to connect'
|
|
error = exc.args
|
|
except Exception:
|
|
pass
|
|
raise zapi.NaApiError(msg, error)
|
|
except Exception as exc:
|
|
raise zapi.NaApiError('Unexpected error', repr(exc))
|
|
|
|
response_xml = response.read()
|
|
response_element = self._get_result(response_xml)
|
|
|
|
if self._trace:
|
|
zapi.LOG.debug("Response: %s", response_element.to_string(pretty=True))
|
|
|
|
return response_element
|
|
|
|
|
|
class OntapRestAPI(object):
|
|
''' wrapper to send requests to ONTAP REST APIs '''
|
|
def __init__(self, module, timeout=60, host_options=None):
|
|
self.host_options = module.params if host_options is None else host_options
|
|
self.module = module
|
|
self.username = self.host_options['username']
|
|
self.password = self.host_options['password']
|
|
self.hostname = self.host_options['hostname']
|
|
self.use_rest = self.host_options['use_rest'].lower()
|
|
self.cert_filepath = self.host_options['cert_filepath']
|
|
self.key_filepath = self.host_options['key_filepath']
|
|
self.verify = self.host_options['validate_certs']
|
|
self.timeout = timeout
|
|
port = self.host_options['http_port']
|
|
if port is None:
|
|
self.url = 'https://%s/api/' % self.hostname
|
|
else:
|
|
self.url = 'https://%s:%d/api/' % (self.hostname, port)
|
|
self.is_rest_error = None
|
|
self.ontap_version = dict(
|
|
full='unknown',
|
|
generation=-1,
|
|
major=-1,
|
|
minor=-1,
|
|
valid=False
|
|
)
|
|
self.errors = []
|
|
self.debug_logs = []
|
|
self.auth_method = set_auth_method(self.module, self.username, self.password, self.cert_filepath, self.key_filepath)
|
|
self.check_required_library()
|
|
if has_feature(module, 'trace_apis'):
|
|
logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG, format='%(asctime)s %(levelname)-8s %(message)s')
|
|
|
|
def requires_ontap_9_6(self, module_name):
|
|
return self.requires_ontap_version(module_name)
|
|
|
|
def requires_ontap_version(self, module_name, version='9.6'):
|
|
suffix = " - %s" % self.is_rest_error if self.is_rest_error is not None else ""
|
|
return "%s only supports REST, and requires ONTAP %s or later.%s" % (module_name, version, suffix)
|
|
|
|
def options_require_ontap_version(self, options, version='9.6', use_rest=None):
|
|
current_version = self.get_ontap_version()
|
|
suffix = " - %s" % self.is_rest_error if self.is_rest_error is not None else ""
|
|
if current_version != (-1, -1, -1):
|
|
suffix += " - ONTAP version: %s.%s.%s" % current_version
|
|
if use_rest is not None:
|
|
suffix += " - using %s" % ('REST' if use_rest else 'ZAPI')
|
|
if isinstance(options, list) and len(options) > 1:
|
|
tag = "any of %s" % options
|
|
elif isinstance(options, list) and len(options) == 1:
|
|
tag = str(options[0])
|
|
else:
|
|
tag = str(options)
|
|
return 'using %s requires ONTAP %s or later and REST must be enabled%s.' % (tag, version, suffix)
|
|
|
|
def meets_rest_minimum_version(self, use_rest, minimum_generation, minimum_major, minimum_minor=0):
|
|
return use_rest and self.get_ontap_version() >= (minimum_generation, minimum_major, minimum_minor)
|
|
|
|
def fail_if_not_rest_minimum_version(self, module_name, minimum_generation, minimum_major, minimum_minor=0):
|
|
status_code = self.get_ontap_version_using_rest()
|
|
msgs = []
|
|
if self.use_rest == 'never':
|
|
msgs.append('Error: REST is required for this module, found: "use_rest: %s".' % self.use_rest)
|
|
if self.is_rest_error:
|
|
msgs.append('Error using REST for version, error: %s.' % self.is_rest_error)
|
|
if status_code != 200:
|
|
msgs.append('Error using REST for version, status_code: %s.' % status_code)
|
|
if msgs:
|
|
self.module.fail_json(msg=' '.join(msgs))
|
|
version = self.get_ontap_version()
|
|
if version < (minimum_generation, minimum_major, minimum_minor):
|
|
msg = 'Error: ' + self.requires_ontap_version(module_name, '%d.%d.%d' % (minimum_generation, minimum_major, minimum_minor))
|
|
msg += ' Found: %s.%s.%s.' % version
|
|
self.module.fail_json(msg=msg)
|
|
|
|
def check_required_library(self):
|
|
if not HAS_REQUESTS:
|
|
self.module.fail_json(msg=missing_required_lib('requests'))
|
|
|
|
def build_headers(self, accept=None, vserver_name=None, vserver_uuid=None):
|
|
headers = {'X-Dot-Client-App': CLIENT_APP_VERSION % self.module._name}
|
|
# accept is used to turn on/off HAL linking
|
|
if accept is not None:
|
|
headers['accept'] = accept
|
|
# vserver tunneling using vserver name and/or UUID
|
|
if vserver_name is not None:
|
|
headers['X-Dot-SVM-Name'] = vserver_name
|
|
if vserver_uuid is not None:
|
|
headers['X-Dot-SVM-UUID'] = vserver_uuid
|
|
return headers
|
|
|
|
def send_request(self, method, api, params, json=None, headers=None):
|
|
''' send http request and process reponse, including error conditions '''
|
|
url = self.url + api
|
|
|
|
def get_auth_args():
|
|
if self.auth_method == 'single_cert':
|
|
kwargs = dict(cert=self.cert_filepath)
|
|
elif self.auth_method == 'cert_key':
|
|
kwargs = dict(cert=(self.cert_filepath, self.key_filepath))
|
|
elif self.auth_method in ('basic_auth', 'speedy_basic_auth'):
|
|
# with requests, there is no challenge, eg no 401.
|
|
kwargs = dict(auth=(self.username, self.password))
|
|
else:
|
|
raise KeyError(self.auth_method)
|
|
return kwargs
|
|
|
|
status_code, json_dict, error_details = self._send_request(method, url, params, json, headers, get_auth_args())
|
|
|
|
return status_code, json_dict, error_details
|
|
|
|
def _send_request(self, method, url, params, json, headers, auth_args):
|
|
status_code = None
|
|
json_dict = None
|
|
json_error = None
|
|
error_details = None
|
|
if headers is None:
|
|
headers = self.build_headers()
|
|
|
|
def fail_on_non_empty_value(response):
|
|
'''json() may fail on an empty value, but it's OK if no response is expected.
|
|
To avoid false positives, only report an issue when we expect to read a value.
|
|
The first get will see it.
|
|
'''
|
|
if method == 'GET' and has_feature(self.module, 'strict_json_check'):
|
|
contents = response.content
|
|
if len(contents) > 0:
|
|
raise ValueError("Expecting json, got: %s" % contents)
|
|
|
|
def get_json(response):
|
|
''' extract json, and error message if present '''
|
|
try:
|
|
json = response.json()
|
|
except ValueError:
|
|
fail_on_non_empty_value(response)
|
|
return None, None
|
|
return json, json.get('error')
|
|
|
|
self.log_debug('sending', repr(dict(method=method, url=url, verify=self.verify, params=params,
|
|
timeout=self.timeout, json=json, headers=headers, auth_args=auth_args)))
|
|
try:
|
|
response = requests.request(method, url, verify=self.verify, params=params,
|
|
timeout=self.timeout, json=json, headers=headers, **auth_args)
|
|
self.log_debug(status_code, response.content)
|
|
status_code = response.status_code
|
|
# If the response was successful, no Exception will be raised
|
|
response.raise_for_status()
|
|
json_dict, json_error = get_json(response)
|
|
except requests.exceptions.HTTPError as err:
|
|
try:
|
|
__, json_error = get_json(response)
|
|
except (AttributeError, ValueError):
|
|
json_error = None
|
|
if json_error is None:
|
|
self.log_error(status_code, 'HTTP error: %s' % err)
|
|
error_details = str(err)
|
|
|
|
# If an error was reported in the json payload, it is handled below
|
|
except requests.exceptions.ConnectionError as err:
|
|
self.log_error(status_code, 'Connection error: %s' % err)
|
|
error_details = str(err)
|
|
except Exception as err:
|
|
self.log_error(status_code, 'Other error: %s' % err)
|
|
error_details = str(err)
|
|
if json_error is not None:
|
|
self.log_error(status_code, 'Endpoint error: %d: %s' % (status_code, json_error))
|
|
error_details = json_error
|
|
if not error_details and not json_dict and method == 'OPTIONS':
|
|
# OPTIONS provides the list of supported verbs
|
|
json_dict['Allow'] = response.headers.get('Allow')
|
|
return status_code, json_dict, error_details
|
|
|
|
def wait_on_job(self, job, timeout=600, increment=60):
|
|
try:
|
|
url = job['_links']['self']['href'].split('api/')[1]
|
|
except Exception as err:
|
|
self.log_error(0, 'URL Incorrect format: %s - Job: %s' % (err, job))
|
|
return None, 'URL Incorrect format: %s - Job: %s' % (err, job)
|
|
# Expecting job to be in the following format
|
|
# {'job':
|
|
# {'uuid': 'fde79888-692a-11ea-80c2-005056b39fe7',
|
|
# '_links':
|
|
# {'self':
|
|
# {'href': '/api/cluster/jobs/fde79888-692a-11ea-80c2-005056b39fe7'}
|
|
# }
|
|
# }
|
|
# }
|
|
error = None
|
|
message = None
|
|
runtime = 0
|
|
retries = 0
|
|
max_retries = 3
|
|
while True:
|
|
# Will run every every <increment> seconds for <timeout> seconds
|
|
job_json, job_error = self.get(url, None)
|
|
job_state = job_json.get('state', None) if job_json else None
|
|
# ignore error if status is provided in the job
|
|
if job_error and job_state is None:
|
|
error = job_error
|
|
retries += 1
|
|
if retries > max_retries:
|
|
self.log_error(0, 'Job error: Reach max retries.')
|
|
break
|
|
else:
|
|
retries = 0
|
|
# a job looks like this
|
|
# {
|
|
# "uuid": "cca3d070-58c6-11ea-8c0c-005056826c14",
|
|
# "description": "POST /api/cluster/metrocluster",
|
|
# "state": "failure",
|
|
# "message": "There are not enough disks in Pool1.", **OPTIONAL**
|
|
# "code": 2432836,
|
|
# "start_time": "2020-02-26T10:35:44-08:00",
|
|
# "end_time": "2020-02-26T10:47:38-08:00",
|
|
# "_links": {
|
|
# "self": {
|
|
# "href": "/api/cluster/jobs/cca3d070-58c6-11ea-8c0c-005056826c14"
|
|
# }
|
|
# }
|
|
# }
|
|
|
|
message = job_json.get('message', '') if job_json else None
|
|
if job_state == 'failure':
|
|
# if the job has failed, return message as error
|
|
return None, message
|
|
if job_state not in ('queued', 'running', None):
|
|
if error is None:
|
|
error = job_error
|
|
break
|
|
elif runtime >= timeout:
|
|
# Would like to post a message to user (not sure how)
|
|
if job_state != 'success':
|
|
self.log_error(0, 'Timeout error: Process still running')
|
|
if error is None:
|
|
error = 'Timeout error: Process still running'
|
|
if job_error is not None:
|
|
error += ' - %s' % job_error
|
|
break
|
|
time.sleep(increment)
|
|
runtime += increment
|
|
return message, error
|
|
|
|
def get(self, api, params=None, headers=None):
|
|
method = 'GET'
|
|
dummy, message, error = self.send_request(method, api, params, headers=headers)
|
|
return message, error
|
|
|
|
def post(self, api, body, params=None, headers=None):
|
|
method = 'POST'
|
|
dummy, message, error = self.send_request(method, api, params, json=body, headers=headers)
|
|
return message, error
|
|
|
|
def patch(self, api, body, params=None, headers=None):
|
|
method = 'PATCH'
|
|
dummy, message, error = self.send_request(method, api, params, json=body, headers=headers)
|
|
return message, error
|
|
|
|
def delete(self, api, body=None, params=None, headers=None):
|
|
method = 'DELETE'
|
|
dummy, message, error = self.send_request(method, api, params, json=body, headers=headers)
|
|
return message, error
|
|
|
|
def options(self, api, params=None, headers=None):
|
|
method = 'OPTIONS'
|
|
dummy, message, error = self.send_request(method, api, params, headers=headers)
|
|
return message, error
|
|
|
|
def set_version(self, message):
|
|
try:
|
|
version = message.get('version', 'not found')
|
|
except AttributeError:
|
|
self.ontap_version['valid'] = False
|
|
self.ontap_version['full'] = 'unreadable message'
|
|
return
|
|
for key in self.ontap_version:
|
|
try:
|
|
self.ontap_version[key] = version.get(key, -1)
|
|
except AttributeError:
|
|
self.ontap_version[key] = -1
|
|
self.ontap_version['valid'] = all(
|
|
self.ontap_version[key] != -1 for key in self.ontap_version if key != 'valid'
|
|
)
|
|
|
|
def get_ontap_version(self):
|
|
if self.ontap_version['valid']:
|
|
return self.ontap_version['generation'], self.ontap_version['major'], self.ontap_version['minor']
|
|
return -1, -1, -1
|
|
|
|
def get_node_version_using_rest(self):
|
|
# using GET rather than HEAD because the error messages are different,
|
|
# and we need the version as some REST options are not available in earlier versions
|
|
method = 'GET'
|
|
api = 'cluster/nodes'
|
|
params = {'fields': ['version']}
|
|
status_code, message, error = self.send_request(method, api, params=params)
|
|
if message and 'records' in message and len(message['records']) > 0:
|
|
message = message['records'][0]
|
|
return status_code, message, error
|
|
|
|
def get_ontap_version_using_rest(self):
|
|
# using GET rather than HEAD because the error messages are different,
|
|
# and we need the version as some REST options are not available in earlier versions
|
|
method = 'GET'
|
|
api = 'cluster'
|
|
params = {'fields': ['version']}
|
|
status_code, message, error = self.send_request(method, api, params=params)
|
|
try:
|
|
if error and 'are available in precluster.' in error.get('message', ''):
|
|
# in precluster mode, version is not available :(
|
|
status_code, message, error = self.get_node_version_using_rest()
|
|
except AttributeError:
|
|
pass
|
|
self.set_version(message)
|
|
self.is_rest_error = str(error) if error else None
|
|
if error:
|
|
self.log_error(status_code, str(error))
|
|
return status_code
|
|
|
|
def _is_rest(self, used_unsupported_rest_properties=None, partially_supported_rest_properties=None, parameters=None):
|
|
if self.use_rest not in ['always', 'auto', 'never']:
|
|
error = "use_rest must be one of: never, always, auto. Got: '%s'" % self.use_rest
|
|
return False, error
|
|
if self.use_rest == "always" and used_unsupported_rest_properties:
|
|
error = "REST API currently does not support '%s'" % \
|
|
', '.join(used_unsupported_rest_properties)
|
|
return True, error
|
|
if self.use_rest == 'never':
|
|
# force ZAPI if requested
|
|
return False, None
|
|
status_code = self.get_ontap_version_using_rest()
|
|
if self.use_rest == "always" and partially_supported_rest_properties:
|
|
error = ''.join(
|
|
"Minimum version of ONTAP for %s is %s\n"
|
|
% (property[0], str(property[1]))
|
|
for property in partially_supported_rest_properties
|
|
if self.get_ontap_version()[0:3] < property[1] and property[0] in parameters
|
|
)
|
|
if error != '':
|
|
return True, error
|
|
if self.use_rest == 'always':
|
|
# ignore error, it will show up later when calling another REST API
|
|
return True, None
|
|
# we're now using 'auto'
|
|
if used_unsupported_rest_properties:
|
|
# force ZAPI if some parameter requires it
|
|
if self.get_ontap_version()[:2] > (9, 5):
|
|
self.module.warn('Falling back to ZAPI because of unsupported option(s) or option value(s) in REST: %s' % used_unsupported_rest_properties)
|
|
return False, None
|
|
if partially_supported_rest_properties:
|
|
# if ontap version is lower than partially_supported_rest_properties version, force ZAPI, only if the paramater is used
|
|
for property in partially_supported_rest_properties:
|
|
if self.get_ontap_version()[0:2] < property[1] and property[0] in parameters:
|
|
self.module.warn(
|
|
'Falling back to ZAPI because of unsupported option(s) or option value(s) "%s" in REST require %s' % (property[0], str(property[1])))
|
|
return False, None
|
|
if self.get_ontap_version()[:2] in ((9, 4), (9, 5)):
|
|
# we can't trust REST support on 9.5, and not at all on 9.4
|
|
return False, None
|
|
if status_code == 200:
|
|
return True, None
|
|
return False, None
|
|
|
|
def is_rest_supported_properties(self, parameters, unsupported_rest_properties=None, partially_supported_rest_properties=None):
|
|
used_unsupported_rest_properties = None
|
|
if unsupported_rest_properties:
|
|
used_unsupported_rest_properties = [x for x in unsupported_rest_properties if x in parameters]
|
|
use_rest, error = self.is_rest(used_unsupported_rest_properties, partially_supported_rest_properties, parameters)
|
|
if error:
|
|
self.module.fail_json(msg=error)
|
|
return use_rest
|
|
|
|
def is_rest(self, used_unsupported_rest_properties=None, partially_supported_rest_properties=None, parameters=None):
|
|
''' only return error if there is a reason to '''
|
|
use_rest, error = self._is_rest(used_unsupported_rest_properties, partially_supported_rest_properties, parameters)
|
|
if used_unsupported_rest_properties is None:
|
|
return use_rest
|
|
return use_rest, error
|
|
|
|
def log_error(self, status_code, message):
|
|
LOG.error("%s: %s", status_code, message)
|
|
self.errors.append(message)
|
|
self.debug_logs.append((status_code, message))
|
|
|
|
def log_debug(self, status_code, content):
|
|
LOG.debug("%s: %s", status_code, content)
|
|
self.debug_logs.append((status_code, content))
|
|
|
|
def write_to_file(self, tag, data=None, filepath=None, append=True):
|
|
'''
|
|
This function is only for debug purposes, all calls to write_to_file should be removed
|
|
before submitting.
|
|
If data is None, tag is considered as data
|
|
else tag is a label, and data is data.
|
|
'''
|
|
if filepath is None:
|
|
filepath = '/tmp/ontap_log'
|
|
mode = 'a' if append else 'w'
|
|
with open(filepath, mode) as afile:
|
|
if data is not None:
|
|
afile.write("%s: %s\n" % (str(tag), str(data)))
|
|
else:
|
|
afile.write(str(tag))
|
|
afile.write('\n')
|
|
|
|
def write_errors_to_file(self, tag=None, filepath=None, append=True):
|
|
if tag is None:
|
|
tag = 'Error'
|
|
for error in self.errors:
|
|
self.write_to_file(tag, error, filepath, append)
|
|
if not append:
|
|
append = True
|
|
|
|
def write_debug_log_to_file(self, tag=None, filepath=None, append=True):
|
|
if tag is None:
|
|
tag = 'Debug'
|
|
for status_code, message in self.debug_logs:
|
|
self.write_to_file(tag, status_code, filepath, append)
|
|
if not append:
|
|
append = True
|
|
self.write_to_file(tag, message, filepath, append)
|