=== modified file 'charmhelpers/contrib/openstack/amulet/deployment.py'
--- charmhelpers/contrib/openstack/amulet/deployment.py 2015-10-22 13:25:25 +0000
+++ charmhelpers/contrib/openstack/amulet/deployment.py 2016-01-07 14:34:44 +0000
@@ -14,12 +14,18 @@
# You should have received a copy of the GNU Lesser General Public License
# along with charm-helpers. If not, see .
+import logging
+import re
+import sys
import six
from collections import OrderedDict
from charmhelpers.contrib.amulet.deployment import (
AmuletDeployment
)
+DEBUG = logging.DEBUG
+ERROR = logging.ERROR
+
class OpenStackAmuletDeployment(AmuletDeployment):
"""OpenStack amulet deployment.
@@ -28,9 +34,12 @@
that is specifically for use by OpenStack charms.
"""
- def __init__(self, series=None, openstack=None, source=None, stable=True):
+ def __init__(self, series=None, openstack=None, source=None,
+ stable=True, log_level=DEBUG):
"""Initialize the deployment environment."""
super(OpenStackAmuletDeployment, self).__init__(series)
+ self.log = self.get_logger(level=log_level)
+ self.log.info('OpenStackAmuletDeployment: init')
self.openstack = openstack
self.source = source
self.stable = stable
@@ -38,6 +47,22 @@
# out.
self.current_next = "trusty"
+ def get_logger(self, name="deployment-logger", level=logging.DEBUG):
+ """Get a logger object that will log to stdout."""
+ log = logging
+ logger = log.getLogger(name)
+ fmt = log.Formatter("%(asctime)s %(funcName)s "
+ "%(levelname)s: %(message)s")
+
+ handler = log.StreamHandler(stream=sys.stdout)
+ handler.setLevel(level)
+ handler.setFormatter(fmt)
+
+ logger.addHandler(handler)
+ logger.setLevel(level)
+
+ return logger
+
def _determine_branch_locations(self, other_services):
"""Determine the branch locations for the other services.
@@ -45,6 +70,8 @@
stable or next (dev) branch, and based on this, use the corresonding
stable or next branches for the other_services."""
+ self.log.info('OpenStackAmuletDeployment: determine branch locations')
+
# Charms outside the lp:~openstack-charmers namespace
base_charms = ['mysql', 'mongodb', 'nrpe']
@@ -82,6 +109,8 @@
def _add_services(self, this_service, other_services):
"""Add services to the deployment and set openstack-origin/source."""
+ self.log.info('OpenStackAmuletDeployment: adding services')
+
other_services = self._determine_branch_locations(other_services)
super(OpenStackAmuletDeployment, self)._add_services(this_service,
@@ -111,9 +140,79 @@
def _configure_services(self, configs):
"""Configure all of the services."""
+ self.log.info('OpenStackAmuletDeployment: configure services')
for service, config in six.iteritems(configs):
self.d.configure(service, config)
+ def _auto_wait_for_status(self, message=None, exclude_services=None,
+ include_only=None, timeout=1800):
+ """Wait for all units to have a specific extended status, except
+ for any defined as excluded. Unless specified via message, any
+ status containing any case of 'ready' will be considered a match.
+
+ Examples of message usage:
+
+ Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
+ message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
+
+ Wait for all units to reach this status (exact match):
+ message = re.compile('^Unit is ready and clustered$')
+
+ Wait for all units to reach any one of these (exact match):
+ message = re.compile('Unit is ready|OK|Ready')
+
+ Wait for at least one unit to reach this status (exact match):
+ message = {'ready'}
+
+ See Amulet's sentry.wait_for_messages() for message usage detail.
+ https://github.com/juju/amulet/blob/master/amulet/sentry.py
+
+ :param message: Expected status match
+ :param exclude_services: List of juju service names to ignore,
+ not to be used in conjuction with include_only.
+ :param include_only: List of juju service names to exclusively check,
+ not to be used in conjuction with exclude_services.
+ :param timeout: Maximum time in seconds to wait for status match
+ :returns: None. Raises if timeout is hit.
+ """
+ self.log.info('Waiting for extended status on units...')
+
+ all_services = self.d.services.keys()
+
+ if exclude_services and include_only:
+ raise ValueError('exclude_services can not be used '
+ 'with include_only')
+
+ if message:
+ if isinstance(message, re._pattern_type):
+ match = message.pattern
+ else:
+ match = message
+
+ self.log.debug('Custom extended status wait match: '
+ '{}'.format(match))
+ else:
+ self.log.debug('Default extended status wait match: contains '
+ 'READY (case-insensitive)')
+ message = re.compile('.*ready.*', re.IGNORECASE)
+
+ if exclude_services:
+ self.log.debug('Excluding services from extended status match: '
+ '{}'.format(exclude_services))
+ else:
+ exclude_services = []
+
+ if include_only:
+ services = include_only
+ else:
+ services = list(set(all_services) - set(exclude_services))
+
+ self.log.debug('Waiting up to {}s for extended status on services: '
+ '{}'.format(timeout, services))
+ service_messages = {service: message for service in services}
+ self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+ self.log.info('OK')
+
def _get_openstack_release(self):
"""Get openstack release.
=== modified file 'charmhelpers/contrib/openstack/amulet/utils.py'
--- charmhelpers/contrib/openstack/amulet/utils.py 2015-10-22 13:25:25 +0000
+++ charmhelpers/contrib/openstack/amulet/utils.py 2016-01-07 14:34:44 +0000
@@ -18,6 +18,7 @@
import json
import logging
import os
+import re
import six
import time
import urllib
@@ -604,7 +605,22 @@
'{}'.format(sample_type, samples))
return None
-# rabbitmq/amqp specific helpers:
+ # rabbitmq/amqp specific helpers:
+
+ def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
+ """Wait for rmq units extended status to show cluster readiness,
+ after an optional initial sleep period. Initial sleep is likely
+ necessary to be effective following a config change, as status
+ message may not instantly update to non-ready."""
+
+ if init_sleep:
+ time.sleep(init_sleep)
+
+ message = re.compile('^Unit is ready and clustered$')
+ deployment._auto_wait_for_status(message=message,
+ timeout=timeout,
+ include_only=['rabbitmq-server'])
+
def add_rmq_test_user(self, sentry_units,
username="testuser1", password="changeme"):
"""Add a test user via the first rmq juju unit, check connection as
@@ -805,7 +821,10 @@
if port:
config['ssl_port'] = port
- deployment.configure('rabbitmq-server', config)
+ deployment.d.configure('rabbitmq-server', config)
+
+ # Wait for unit status
+ self.rmq_wait_for_cluster(deployment)
# Confirm
tries = 0
@@ -832,7 +851,10 @@
# Disable RMQ SSL
config = {'ssl': 'off'}
- deployment.configure('rabbitmq-server', config)
+ deployment.d.configure('rabbitmq-server', config)
+
+ # Wait for unit status
+ self.rmq_wait_for_cluster(deployment)
# Confirm
tries = 0
=== modified file 'charmhelpers/contrib/openstack/utils.py'
--- charmhelpers/contrib/openstack/utils.py 2015-10-22 13:25:25 +0000
+++ charmhelpers/contrib/openstack/utils.py 2016-01-07 14:34:44 +0000
@@ -127,31 +127,31 @@
# >= Liberty version->codename mapping
PACKAGE_CODENAMES = {
'nova-common': OrderedDict([
- ('12.0.0', 'liberty'),
+ ('12.0', 'liberty'),
]),
'neutron-common': OrderedDict([
- ('7.0.0', 'liberty'),
+ ('7.0', 'liberty'),
]),
'cinder-common': OrderedDict([
- ('7.0.0', 'liberty'),
+ ('7.0', 'liberty'),
]),
'keystone': OrderedDict([
- ('8.0.0', 'liberty'),
+ ('8.0', 'liberty'),
]),
'horizon-common': OrderedDict([
- ('8.0.0', 'liberty'),
+ ('8.0', 'liberty'),
]),
'ceilometer-common': OrderedDict([
- ('5.0.0', 'liberty'),
+ ('5.0', 'liberty'),
]),
'heat-common': OrderedDict([
- ('5.0.0', 'liberty'),
+ ('5.0', 'liberty'),
]),
'glance-common': OrderedDict([
- ('11.0.0', 'liberty'),
+ ('11.0', 'liberty'),
]),
'openstack-dashboard': OrderedDict([
- ('8.0.0', 'liberty'),
+ ('8.0', 'liberty'),
]),
}
@@ -238,7 +238,14 @@
error_out(e)
vers = apt.upstream_version(pkg.current_ver.ver_str)
- match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
+ if 'swift' in pkg.name:
+ # Fully x.y.z match for swift versions
+ match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
+ else:
+ # x.y match only for 20XX.X
+ # and ignore patch level for other packages
+ match = re.match('^(\d+)\.(\d+)', vers)
+
if match:
vers = match.group(0)
@@ -250,13 +257,8 @@
# < Liberty co-ordinated project versions
try:
if 'swift' in pkg.name:
- swift_vers = vers[:5]
- if swift_vers not in SWIFT_CODENAMES:
- # Deal with 1.10.0 upward
- swift_vers = vers[:6]
- return SWIFT_CODENAMES[swift_vers]
+ return SWIFT_CODENAMES[vers]
else:
- vers = vers[:6]
return OPENSTACK_CODENAMES[vers]
except KeyError:
if not fatal:
@@ -859,7 +861,9 @@
if charm_state != 'active' and charm_state != 'unknown':
state = workload_state_compare(state, charm_state)
if message:
- message = "{} {}".format(message, charm_message)
+ charm_message = charm_message.replace("Incomplete relations: ",
+ "")
+ message = "{}, {}".format(message, charm_message)
else:
message = charm_message
=== modified file 'charmhelpers/contrib/storage/linux/loopback.py'
--- charmhelpers/contrib/storage/linux/loopback.py 2015-11-17 17:26:08 +0000
+++ charmhelpers/contrib/storage/linux/loopback.py 2016-01-07 14:34:44 +0000
@@ -76,13 +76,3 @@
check_call(cmd)
return create_loopback(path)
-
-
-def is_mapped_loopback_device(device):
- """
- Checks if a given device name is an existing/mapped loopback device.
- :param device: str: Full path to the device (eg, /dev/loop1).
- :returns: str: Path to the backing file if is a loopback device
- empty string otherwise
- """
- return loopback_devices().get(device, "")
=== modified file 'charmhelpers/core/host.py'
--- charmhelpers/core/host.py 2015-10-22 13:25:25 +0000
+++ charmhelpers/core/host.py 2016-01-07 14:34:44 +0000
@@ -566,7 +566,14 @@
os.chdir(cur)
-def chownr(path, owner, group, follow_links=True):
+def chownr(path, owner, group, follow_links=True, chowntopdir=False):
+ """
+ Recursively change user and group ownership of files and directories
+ in given path. Doesn't chown path itself by default, only its children.
+
+ :param bool follow_links: Also Chown links if True
+ :param bool chowntopdir: Also chown path itself if True
+ """
uid = pwd.getpwnam(owner).pw_uid
gid = grp.getgrnam(group).gr_gid
if follow_links:
@@ -574,6 +581,10 @@
else:
chown = os.lchown
+ if chowntopdir:
+ broken_symlink = os.path.lexists(path) and not os.path.exists(path)
+ if not broken_symlink:
+ chown(path, uid, gid)
for root, dirs, files in os.walk(path):
for name in dirs + files:
full = os.path.join(root, name)
=== modified file 'charmhelpers/core/hugepage.py'
--- charmhelpers/core/hugepage.py 2015-10-22 13:25:25 +0000
+++ charmhelpers/core/hugepage.py 2016-01-07 14:34:44 +0000
@@ -46,6 +46,8 @@
group_info = add_group(group)
gid = group_info.gr_gid
add_user_to_group(user, group)
+ if max_map_count < 2 * nr_hugepages:
+ max_map_count = 2 * nr_hugepages
sysctl_settings = {
'vm.nr_hugepages': nr_hugepages,
'vm.max_map_count': max_map_count,
=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
--- tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-10-22 13:25:25 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py 2016-01-07 14:34:44 +0000
@@ -14,12 +14,18 @@
# You should have received a copy of the GNU Lesser General Public License
# along with charm-helpers. If not, see .
+import logging
+import re
+import sys
import six
from collections import OrderedDict
from charmhelpers.contrib.amulet.deployment import (
AmuletDeployment
)
+DEBUG = logging.DEBUG
+ERROR = logging.ERROR
+
class OpenStackAmuletDeployment(AmuletDeployment):
"""OpenStack amulet deployment.
@@ -28,9 +34,12 @@
that is specifically for use by OpenStack charms.
"""
- def __init__(self, series=None, openstack=None, source=None, stable=True):
+ def __init__(self, series=None, openstack=None, source=None,
+ stable=True, log_level=DEBUG):
"""Initialize the deployment environment."""
super(OpenStackAmuletDeployment, self).__init__(series)
+ self.log = self.get_logger(level=log_level)
+ self.log.info('OpenStackAmuletDeployment: init')
self.openstack = openstack
self.source = source
self.stable = stable
@@ -38,6 +47,22 @@
# out.
self.current_next = "trusty"
+ def get_logger(self, name="deployment-logger", level=logging.DEBUG):
+ """Get a logger object that will log to stdout."""
+ log = logging
+ logger = log.getLogger(name)
+ fmt = log.Formatter("%(asctime)s %(funcName)s "
+ "%(levelname)s: %(message)s")
+
+ handler = log.StreamHandler(stream=sys.stdout)
+ handler.setLevel(level)
+ handler.setFormatter(fmt)
+
+ logger.addHandler(handler)
+ logger.setLevel(level)
+
+ return logger
+
def _determine_branch_locations(self, other_services):
"""Determine the branch locations for the other services.
@@ -45,6 +70,8 @@
stable or next (dev) branch, and based on this, use the corresonding
stable or next branches for the other_services."""
+ self.log.info('OpenStackAmuletDeployment: determine branch locations')
+
# Charms outside the lp:~openstack-charmers namespace
base_charms = ['mysql', 'mongodb', 'nrpe']
@@ -82,6 +109,8 @@
def _add_services(self, this_service, other_services):
"""Add services to the deployment and set openstack-origin/source."""
+ self.log.info('OpenStackAmuletDeployment: adding services')
+
other_services = self._determine_branch_locations(other_services)
super(OpenStackAmuletDeployment, self)._add_services(this_service,
@@ -111,9 +140,79 @@
def _configure_services(self, configs):
"""Configure all of the services."""
+ self.log.info('OpenStackAmuletDeployment: configure services')
for service, config in six.iteritems(configs):
self.d.configure(service, config)
+ def _auto_wait_for_status(self, message=None, exclude_services=None,
+ include_only=None, timeout=1800):
+ """Wait for all units to have a specific extended status, except
+ for any defined as excluded. Unless specified via message, any
+ status containing any case of 'ready' will be considered a match.
+
+ Examples of message usage:
+
+ Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
+ message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
+
+ Wait for all units to reach this status (exact match):
+ message = re.compile('^Unit is ready and clustered$')
+
+ Wait for all units to reach any one of these (exact match):
+ message = re.compile('Unit is ready|OK|Ready')
+
+ Wait for at least one unit to reach this status (exact match):
+ message = {'ready'}
+
+ See Amulet's sentry.wait_for_messages() for message usage detail.
+ https://github.com/juju/amulet/blob/master/amulet/sentry.py
+
+ :param message: Expected status match
+ :param exclude_services: List of juju service names to ignore,
+ not to be used in conjuction with include_only.
+ :param include_only: List of juju service names to exclusively check,
+ not to be used in conjuction with exclude_services.
+ :param timeout: Maximum time in seconds to wait for status match
+ :returns: None. Raises if timeout is hit.
+ """
+ self.log.info('Waiting for extended status on units...')
+
+ all_services = self.d.services.keys()
+
+ if exclude_services and include_only:
+ raise ValueError('exclude_services can not be used '
+ 'with include_only')
+
+ if message:
+ if isinstance(message, re._pattern_type):
+ match = message.pattern
+ else:
+ match = message
+
+ self.log.debug('Custom extended status wait match: '
+ '{}'.format(match))
+ else:
+ self.log.debug('Default extended status wait match: contains '
+ 'READY (case-insensitive)')
+ message = re.compile('.*ready.*', re.IGNORECASE)
+
+ if exclude_services:
+ self.log.debug('Excluding services from extended status match: '
+ '{}'.format(exclude_services))
+ else:
+ exclude_services = []
+
+ if include_only:
+ services = include_only
+ else:
+ services = list(set(all_services) - set(exclude_services))
+
+ self.log.debug('Waiting up to {}s for extended status on services: '
+ '{}'.format(timeout, services))
+ service_messages = {service: message for service in services}
+ self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+ self.log.info('OK')
+
def _get_openstack_release(self):
"""Get openstack release.
=== modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py'
--- tests/charmhelpers/contrib/openstack/amulet/utils.py 2015-10-22 13:25:25 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/utils.py 2016-01-07 14:34:44 +0000
@@ -18,6 +18,7 @@
import json
import logging
import os
+import re
import six
import time
import urllib
@@ -604,7 +605,22 @@
'{}'.format(sample_type, samples))
return None
-# rabbitmq/amqp specific helpers:
+ # rabbitmq/amqp specific helpers:
+
+ def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
+ """Wait for rmq units extended status to show cluster readiness,
+ after an optional initial sleep period. Initial sleep is likely
+ necessary to be effective following a config change, as status
+ message may not instantly update to non-ready."""
+
+ if init_sleep:
+ time.sleep(init_sleep)
+
+ message = re.compile('^Unit is ready and clustered$')
+ deployment._auto_wait_for_status(message=message,
+ timeout=timeout,
+ include_only=['rabbitmq-server'])
+
def add_rmq_test_user(self, sentry_units,
username="testuser1", password="changeme"):
"""Add a test user via the first rmq juju unit, check connection as
@@ -805,7 +821,10 @@
if port:
config['ssl_port'] = port
- deployment.configure('rabbitmq-server', config)
+ deployment.d.configure('rabbitmq-server', config)
+
+ # Wait for unit status
+ self.rmq_wait_for_cluster(deployment)
# Confirm
tries = 0
@@ -832,7 +851,10 @@
# Disable RMQ SSL
config = {'ssl': 'off'}
- deployment.configure('rabbitmq-server', config)
+ deployment.d.configure('rabbitmq-server', config)
+
+ # Wait for unit status
+ self.rmq_wait_for_cluster(deployment)
# Confirm
tries = 0