# Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Backup manager manages volume backups.

Volume Backups are full copies of persistent volumes stored in a backup
store e.g. an object store or any other backup store if and when support is
added. They are usable without the original object being available. A
volume backup can be restored to the original volume it was created from or
any other available volume with a minimum size of the original volume.
Volume backups can be created, restored, deleted and listed.

**Related Flags**

:backup_manager:  The module name of a class derived from
                          :class:`manager.Manager` (default:
                          :class:`cinder.backup.manager.Manager`).

"""

import os

from castellan import key_manager
from eventlet import tpool
from oslo_config import cfg
from oslo_log import log as logging
from oslo_log import versionutils
import oslo_messaging as messaging
from oslo_service import loopingcall
from oslo_service import periodic_task
from oslo_utils import excutils
from oslo_utils import importutils
import six

from cinder.backup import driver
from cinder.backup import rpcapi as backup_rpcapi
from cinder import context
from cinder import exception
from cinder.i18n import _
from cinder import manager
from cinder import objects
from cinder.objects import fields
from cinder import quota
from cinder import rpc
from cinder import utils
from cinder.volume import rpcapi as volume_rpcapi
from cinder.volume import utils as volume_utils

LOG = logging.getLogger(__name__)

backup_manager_opts = [
    cfg.StrOpt('backup_driver',
               default='cinder.backup.drivers.swift.SwiftBackupDriver',
               help='Driver to use for backups.',),
    cfg.BoolOpt('backup_service_inithost_offload',
                default=True,
                help='Offload pending backup delete during '
                     'backup service startup. If false, the backup service '
                     'will remain down until all pending backups are '
                     'deleted.',),
]

# This map doesn't need to be extended in the future since it's only
# for old backup services
mapper = {'cinder.backup.services.swift': 'cinder.backup.drivers.swift',
          'cinder.backup.services.ceph': 'cinder.backup.drivers.ceph'}

CONF = cfg.CONF
CONF.register_opts(backup_manager_opts)
CONF.import_opt('use_multipath_for_image_xfer', 'cinder.volume.driver')
CONF.import_opt('num_volume_device_scan_tries', 'cinder.volume.driver')
QUOTAS = quota.QUOTAS


# TODO(geguileo): Once Eventlet issue #432 gets fixed we can just tpool.execute
# the whole call to the driver's backup and restore methods instead of proxy
# wrapping the device_file and having the drivers also proxy wrap their
# writes/reads and the compression/decompression calls.
# (https://github.com/eventlet/eventlet/issues/432)

class BackupManager(manager.ThreadPoolManager):
    """Manages backup of block storage devices."""

    RPC_API_VERSION = backup_rpcapi.BackupAPI.RPC_API_VERSION

    target = messaging.Target(version=RPC_API_VERSION)

    def __init__(self, *args, **kwargs):
        self.az = CONF.storage_availability_zone
        self.backup_rpcapi = backup_rpcapi.BackupAPI()
        self.volume_rpcapi = volume_rpcapi.VolumeAPI()
        super(BackupManager, self).__init__(*args, **kwargs)
        self.is_initialized = False

    @property
    def driver_name(self):
        """This function maps old backup services to backup drivers."""

        return self._map_service_to_driver(CONF.backup_driver)

    def _map_service_to_driver(self, service):
        """Maps services to drivers."""

        if service in mapper:
            msg = ("Using legacy backup service configuration like "
                   "cinder.backup.services.* is deprecated and "
                   "will be removed in the 'R' release. Please use "
                   "the cinder.backup.drivers.* method instead.")
            versionutils.report_deprecated_feature(LOG, msg)
            return mapper[service]
        return service

    def get_backup_driver(self, context):
        driver = None
        try:
            # TODO(e0ne): remove backward compatibility in S release
            service = importutils.import_module(self.driver_name)
            msg = ("Backup driver initialization using module name "
                   "is deprecated and will be removed in a 'S' "
                   "release. Please, use classname for backup driver "
                   "reference in the config.")
            versionutils.report_deprecated_feature(LOG, msg)
            driver = service.get_backup_driver(context)
        except ImportError:
            driver_class = importutils.import_class(self.driver_name)
            driver = driver_class(context=context, db=self.db)

        return driver

    def _update_backup_error(self, backup, err,
                             status=fields.BackupStatus.ERROR):
        backup.status = status
        backup.fail_reason = err
        backup.save()

    def init_host(self, **kwargs):
        """Run initialization needed for a standalone service."""
        ctxt = context.get_admin_context()
        self.setup_backup_backend(ctxt)

        try:
            self._cleanup_incomplete_backup_operations(ctxt)
        except Exception:
            # Don't block startup of the backup service.
            LOG.exception("Problem cleaning incomplete backup operations.")

    def _setup_backup_driver(self, ctxt):
        backup_service = self.get_backup_driver(ctxt)
        backup_service.check_for_setup_error()
        self.is_initialized = True
        raise loopingcall.LoopingCallDone()

    def setup_backup_backend(self, ctxt):
        try:
            init_loop = loopingcall.FixedIntervalLoopingCall(
                self._setup_backup_driver, ctxt)
            init_loop.start(interval=CONF.periodic_interval)
        except loopingcall.LoopingCallDone:
            LOG.info("Backup driver was successfully initialized.")
        except Exception:
            LOG.exception("Failed to initialize driver.",
                          resource={'type': 'driver',
                                    'id': self.__class__.__name__})

    def reset(self):
        super(BackupManager, self).reset()
        self.backup_rpcapi = backup_rpcapi.BackupAPI()
        self.volume_rpcapi = volume_rpcapi.VolumeAPI()

    def _cleanup_incomplete_backup_operations(self, ctxt):
        LOG.info("Cleaning up incomplete backup operations.")

        # TODO(smulcahy) implement full resume of backup and restore
        # operations on restart (rather than simply resetting)
        backups = objects.BackupList.get_all_by_host(ctxt, self.host)
        for backup in backups:
            try:
                self._cleanup_one_backup(ctxt, backup)
            except Exception:
                LOG.exception("Problem cleaning up backup %(bkup)s.",
                              {'bkup': backup['id']})
            try:
                self._cleanup_temp_volumes_snapshots_for_one_backup(ctxt,
                                                                    backup)
            except Exception:
                LOG.exception("Problem cleaning temp volumes and "
                              "snapshots for backup %(bkup)s.",
                              {'bkup': backup['id']})

    def _cleanup_one_volume(self, ctxt, volume):
        if volume['status'] == 'backing-up':
            self._detach_all_attachments(ctxt, volume)
            LOG.info('Resetting volume %(vol_id)s to previous '
                     'status %(status)s (was backing-up).',
                     {'vol_id': volume['id'],
                      'status': volume['previous_status']})
            self.db.volume_update(ctxt, volume['id'],
                                  {'status': volume['previous_status']})
        elif volume['status'] == 'restoring-backup':
            self._detach_all_attachments(ctxt, volume)
            LOG.info('Setting volume %s to error_restoring '
                     '(was restoring-backup).', volume['id'])
            self.db.volume_update(ctxt, volume['id'],
                                  {'status': 'error_restoring'})

    def _cleanup_one_backup(self, ctxt, backup):
        if backup['status'] == fields.BackupStatus.CREATING:
            LOG.info('Resetting backup %s to error (was creating).',
                     backup['id'])

            volume = objects.Volume.get_by_id(ctxt, backup.volume_id)
            self._cleanup_one_volume(ctxt, volume)

            err = 'incomplete backup reset on manager restart'
            self._update_backup_error(backup, err)
        elif backup['status'] == fields.BackupStatus.RESTORING:
            LOG.info('Resetting backup %s to '
                     'available (was restoring).',
                     backup['id'])
            volume = objects.Volume.get_by_id(ctxt, backup.restore_volume_id)
            self._cleanup_one_volume(ctxt, volume)

            backup.status = fields.BackupStatus.AVAILABLE
            backup.save()
        elif backup['status'] == fields.BackupStatus.DELETING:
            # Don't resume deleting the backup of an encrypted volume. The
            # admin context won't be sufficient to delete the backup's copy
            # of the encryption key ID (a real user context is required).
            if backup.encryption_key_id is None:
                LOG.info('Resuming delete on backup: %s.', backup.id)
                if CONF.backup_service_inithost_offload:
                    # Offload all the pending backup delete operations to the
                    # threadpool to prevent the main backup service thread
                    # from being blocked.
                    self._add_to_threadpool(self.delete_backup, ctxt, backup)
                else:
                    # Delete backups sequentially
                    self.delete_backup(ctxt, backup)
            else:
                LOG.info('Unable to resume deleting backup of an encrypted '
                         'volume, resetting backup %s to error_deleting '
                         '(was deleting).',
                         backup.id)
                backup.status = fields.BackupStatus.ERROR_DELETING
                backup.save()

    def _detach_all_attachments(self, ctxt, volume):
        attachments = volume['volume_attachment'] or []
        for attachment in attachments:
            if (attachment['attached_host'] == self.host and
                    attachment['instance_uuid'] is None):
                try:
                    rpcapi = self.volume_rpcapi
                    rpcapi.detach_volume(ctxt, volume, attachment['id'])
                except Exception:
                    LOG.exception("Detach attachment %(attach_id)s failed.",
                                  {'attach_id': attachment['id']},
                                  resource=volume)

    def _delete_temp_volume(self, ctxt, backup):
        try:
            temp_volume = objects.Volume.get_by_id(
                ctxt, backup.temp_volume_id)
            self.volume_rpcapi.delete_volume(ctxt, temp_volume)
        except exception.VolumeNotFound:
            LOG.debug("Could not find temp volume %(vol)s to clean up "
                      "for backup %(backup)s.",
                      {'vol': backup.temp_volume_id,
                       'backup': backup.id})
        backup.temp_volume_id = None
        backup.save()

    def _delete_temp_snapshot(self, ctxt, backup):
        try:
            temp_snapshot = objects.Snapshot.get_by_id(
                ctxt, backup.temp_snapshot_id)
            self.volume_rpcapi.delete_snapshot(ctxt, temp_snapshot)
        except exception.SnapshotNotFound:
            LOG.debug("Could not find temp snapshot %(snap)s to clean "
                      "up for backup %(backup)s.",
                      {'snap': backup.temp_snapshot_id,
                       'backup': backup.id})
        backup.temp_snapshot_id = None
        backup.save()

    def _cleanup_temp_volumes_snapshots_for_one_backup(self, ctxt, backup):
        # NOTE(xyang): If the service crashes or gets restarted during the
        # backup operation, there could be temporary volumes or snapshots
        # that are not deleted. Make sure any temporary volumes or snapshots
        # create by the backup job are deleted when service is started.
        if (backup.temp_volume_id
                and backup.status == fields.BackupStatus.ERROR):
            self._delete_temp_volume(ctxt, backup)

        if (backup.temp_snapshot_id
                and backup.status == fields.BackupStatus.ERROR):
            self._delete_temp_snapshot(ctxt, backup)

    def _cleanup_temp_volumes_snapshots_when_backup_created(
            self, ctxt, backup):
        # Delete temp volumes or snapshots when backup creation is completed.
        if backup.temp_volume_id:
            self._delete_temp_volume(ctxt, backup)

        if backup.temp_snapshot_id:
            self._delete_temp_snapshot(ctxt, backup)

    def create_backup(self, context, backup):
        """Create volume backups using configured backup service."""
        volume_id = backup.volume_id
        snapshot_id = backup.snapshot_id
        volume = objects.Volume.get_by_id(context, volume_id)
        snapshot = objects.Snapshot.get_by_id(
            context, snapshot_id) if snapshot_id else None
        previous_status = volume.get('previous_status', None)
        updates = {}
        if snapshot_id:
            log_message = ('Create backup started, backup: %(backup_id)s '
                           'volume: %(volume_id)s snapshot: %(snapshot_id)s.'
                           % {'backup_id': backup.id,
                              'volume_id': volume_id,
                              'snapshot_id': snapshot_id})
        else:
            log_message = ('Create backup started, backup: %(backup_id)s '
                           'volume: %(volume_id)s.'
                           % {'backup_id': backup.id,
                              'volume_id': volume_id})
        LOG.info(log_message)

        self._notify_about_backup_usage(context, backup, "create.start")

        backup.host = self.host
        backup.service = self.driver_name
        backup.availability_zone = self.az
        backup.save()

        expected_status = "backing-up"
        if snapshot_id:
            actual_status = snapshot['status']
            if actual_status != expected_status:
                err = _('Create backup aborted, expected snapshot status '
                        '%(expected_status)s but got %(actual_status)s.') % {
                    'expected_status': expected_status,
                    'actual_status': actual_status,
                }
                self._update_backup_error(backup, err)
                raise exception.InvalidSnapshot(reason=err)
        else:
            actual_status = volume['status']
            if actual_status != expected_status:
                err = _('Create backup aborted, expected volume status '
                        '%(expected_status)s but got %(actual_status)s.') % {
                    'expected_status': expected_status,
                    'actual_status': actual_status,
                }
                self._update_backup_error(backup, err)
                raise exception.InvalidVolume(reason=err)

        expected_status = fields.BackupStatus.CREATING
        actual_status = backup.status
        if actual_status != expected_status:
            err = _('Create backup aborted, expected backup status '
                    '%(expected_status)s but got %(actual_status)s.') % {
                'expected_status': expected_status,
                'actual_status': actual_status,
            }
            self._update_backup_error(backup, err)
            raise exception.InvalidBackup(reason=err)

        try:
            if not self.is_working():
                err = _('Create backup aborted due to backup service is down')
                self._update_backup_error(backup, err)
                raise exception.InvalidBackup(reason=err)
            updates = self._run_backup(context, backup, volume)
        except Exception as err:
            with excutils.save_and_reraise_exception():
                if snapshot_id:
                    snapshot.status = fields.SnapshotStatus.AVAILABLE
                    snapshot.save()
                else:
                    self.db.volume_update(
                        context, volume_id,
                        {'status': previous_status,
                         'previous_status': 'error_backing-up'})
                self._update_backup_error(backup, six.text_type(err))

        # Restore the original status.
        if snapshot_id:
            self.db.snapshot_update(context, snapshot_id,
                                    {'status': fields.BackupStatus.AVAILABLE})
        else:
            self.db.volume_update(context, volume_id,
                                  {'status': previous_status,
                                   'previous_status': 'backing-up'})
        backup.status = fields.BackupStatus.AVAILABLE
        backup.size = volume['size']

        if updates:
            backup.update(updates)
        backup.save()

        # Handle the num_dependent_backups of parent backup when child backup
        # has created successfully.
        if backup.parent_id:
            parent_backup = objects.Backup.get_by_id(context,
                                                     backup.parent_id)
            parent_backup.num_dependent_backups += 1
            parent_backup.save()
        LOG.info('Create backup finished. backup: %s.', backup.id)
        self._notify_about_backup_usage(context, backup, "create.end")

    def _run_backup(self, context, backup, volume):
        # Save a copy of the encryption key ID in case the volume is deleted.
        if (volume.encryption_key_id is not None and
                backup.encryption_key_id is None):
            backup.encryption_key_id = volume_utils.clone_encryption_key(
                context,
                key_manager.API(CONF),
                volume.encryption_key_id)
            backup.save()

        backup_service = self.get_backup_driver(context)

        properties = utils.brick_get_connector_properties()

        # NOTE(geguileo): Not all I/O disk operations properly do greenthread
        # context switching and may end up blocking the greenthread, so we go
        # with native threads proxy-wrapping the device file object.
        try:
            backup_device = self.volume_rpcapi.get_backup_device(context,
                                                                 backup,
                                                                 volume)
            attach_info = self._attach_device(context,
                                              backup_device.device_obj,
                                              properties,
                                              backup_device.is_snapshot)
            try:
                device_path = attach_info['device']['path']
                if (isinstance(device_path, six.string_types) and
                        not os.path.isdir(device_path)):
                    if backup_device.secure_enabled:
                        with open(device_path) as device_file:
                            updates = backup_service.backup(
                                backup, tpool.Proxy(device_file))
                    else:
                        with utils.temporary_chown(device_path):
                            with open(device_path) as device_file:
                                updates = backup_service.backup(
                                    backup, tpool.Proxy(device_file))
                # device_path is already file-like so no need to open it
                else:
                    updates = backup_service.backup(backup,
                                                    tpool.Proxy(device_path))

            finally:
                self._detach_device(context, attach_info,
                                    backup_device.device_obj, properties,
                                    backup_device.is_snapshot, force=True,
                                    ignore_errors=True)
        finally:
            backup = objects.Backup.get_by_id(context, backup.id)
            self._cleanup_temp_volumes_snapshots_when_backup_created(
                context, backup)
        return updates

    def restore_backup(self, context, backup, volume_id):
        """Restore volume backups from configured backup service."""
        LOG.info('Restore backup started, backup: %(backup_id)s '
                 'volume: %(volume_id)s.',
                 {'backup_id': backup.id, 'volume_id': volume_id})

        volume = objects.Volume.get_by_id(context, volume_id)
        self._notify_about_backup_usage(context, backup, "restore.start")

        backup.host = self.host
        backup.save()

        expected_status = 'restoring-backup'
        actual_status = volume['status']
        if actual_status != expected_status:
            err = (_('Restore backup aborted, expected volume status '
                     '%(expected_status)s but got %(actual_status)s.') %
                   {'expected_status': expected_status,
                    'actual_status': actual_status})
            backup.status = fields.BackupStatus.AVAILABLE
            backup.save()
            self.db.volume_update(context, volume_id,
                                  {'status': 'error_restoring'})
            raise exception.InvalidVolume(reason=err)

        expected_status = fields.BackupStatus.RESTORING
        actual_status = backup['status']
        if actual_status != expected_status:
            err = (_('Restore backup aborted: expected backup status '
                     '%(expected_status)s but got %(actual_status)s.') %
                   {'expected_status': expected_status,
                    'actual_status': actual_status})
            self._update_backup_error(backup, err)
            self.db.volume_update(context, volume_id, {'status': 'error'})
            raise exception.InvalidBackup(reason=err)

        if volume['size'] > backup['size']:
            LOG.info('Volume: %(vol_id)s, size: %(vol_size)d is '
                     'larger than backup: %(backup_id)s, '
                     'size: %(backup_size)d, continuing with restore.',
                     {'vol_id': volume['id'],
                      'vol_size': volume['size'],
                      'backup_id': backup['id'],
                      'backup_size': backup['size']})

        backup_service = self._map_service_to_driver(backup['service'])
        configured_service = self.driver_name
        # TODO(tommylikehu): We upgraded the 'driver_name' from module
        # to class name, so we use 'in' here to match two namings,
        # this can be replaced with equal sign during next
        # release (Rocky).
        if backup_service not in configured_service:
            err = _('Restore backup aborted, the backup service currently'
                    ' configured [%(configured_service)s] is not the'
                    ' backup service that was used to create this'
                    ' backup [%(backup_service)s].') % {
                'configured_service': configured_service,
                'backup_service': backup_service,
            }
            backup.status = fields.BackupStatus.AVAILABLE
            backup.save()
            self.db.volume_update(context, volume_id, {'status': 'error'})
            raise exception.InvalidBackup(reason=err)

        try:
            self._run_restore(context, backup, volume)
        except Exception:
            with excutils.save_and_reraise_exception():
                self.db.volume_update(context, volume_id,
                                      {'status': 'error_restoring'})
                backup.status = fields.BackupStatus.AVAILABLE
                backup.save()

        self.db.volume_update(context, volume_id, {'status': 'available'})
        backup.status = fields.BackupStatus.AVAILABLE
        backup.save()
        LOG.info('Restore backup finished, backup %(backup_id)s restored'
                 ' to volume %(volume_id)s.',
                 {'backup_id': backup.id, 'volume_id': volume_id})
        self._notify_about_backup_usage(context, backup, "restore.end")

    def _run_restore(self, context, backup, volume):
        orig_key_id = volume.encryption_key_id
        backup_service = self.get_backup_driver(context)

        properties = utils.brick_get_connector_properties()
        secure_enabled = (
            self.volume_rpcapi.secure_file_operations_enabled(context,
                                                              volume))
        attach_info = self._attach_device(context, volume, properties)

        # NOTE(geguileo): Not all I/O disk operations properly do greenthread
        # context switching and may end up blocking the greenthread, so we go
        # with native threads proxy-wrapping the device file object.
        try:
            device_path = attach_info['device']['path']
            if (isinstance(device_path, six.string_types) and
                    not os.path.isdir(device_path)):
                if secure_enabled:
                    with open(device_path, 'wb') as device_file:
                        backup_service.restore(backup, volume.id,
                                               tpool.Proxy(device_file))
                else:
                    with utils.temporary_chown(device_path):
                        with open(device_path, 'wb') as device_file:
                            backup_service.restore(backup, volume.id,
                                                   tpool.Proxy(device_file))
            # device_path is already file-like so no need to open it
            else:
                backup_service.restore(backup, volume.id,
                                       tpool.Proxy(device_path))
        finally:
            self._detach_device(context, attach_info, volume, properties,
                                force=True)

        # Regardless of whether the restore was successful, do some
        # housekeeping to ensure the restored volume's encryption key ID is
        # unique, and any previous key ID is deleted. Start by fetching fresh
        # info on the restored volume.
        restored_volume = objects.Volume.get_by_id(context, volume.id)
        restored_key_id = restored_volume.encryption_key_id
        if restored_key_id != orig_key_id:
            LOG.info('Updating encryption key ID for volume %(volume_id)s '
                     'from backup %(backup_id)s.',
                     {'volume_id': volume.id, 'backup_id': backup.id})

            key_mgr = key_manager.API(CONF)
            if orig_key_id is not None:
                LOG.debug('Deleting original volume encryption key ID.')
                volume_utils.delete_encryption_key(context,
                                                   key_mgr,
                                                   orig_key_id)

            if backup.encryption_key_id is None:
                # This backup predates the current code that stores the cloned
                # key ID in the backup database. Fortunately, the key ID
                # restored from the backup data _is_ a clone of the original
                # volume's key ID, so grab it.
                LOG.debug('Gleaning backup encryption key ID from metadata.')
                backup.encryption_key_id = restored_key_id
                backup.save()

            # Clone the key ID again to ensure every restored volume has
            # a unique key ID. The volume's key ID should not be the same
            # as the backup.encryption_key_id (the copy made when the backup
            # was first created).
            new_key_id = volume_utils.clone_encryption_key(
                context,
                key_mgr,
                backup.encryption_key_id)
            restored_volume.encryption_key_id = new_key_id
            restored_volume.save()
        else:
            LOG.debug('Encryption key ID for volume %(volume_id)s already '
                      'matches encryption key ID in backup %(backup_id)s.',
                      {'volume_id': volume.id, 'backup_id': backup.id})

    def delete_backup(self, context, backup):
        """Delete volume backup from configured backup service."""
        LOG.info('Delete backup started, backup: %s.', backup.id)

        self._notify_about_backup_usage(context, backup, "delete.start")
        backup.host = self.host
        backup.save()

        expected_status = fields.BackupStatus.DELETING
        actual_status = backup.status
        if actual_status != expected_status:
            err = _('Delete_backup aborted, expected backup status '
                    '%(expected_status)s but got %(actual_status)s.') \
                % {'expected_status': expected_status,
                   'actual_status': actual_status}
            self._update_backup_error(backup, err)
            raise exception.InvalidBackup(reason=err)

        if not self.is_working():
            err = _('Delete backup is aborted due to backup service is down')
            status = fields.BackupStatus.ERROR_DELETING
            self._update_backup_error(backup, err, status)
            raise exception.InvalidBackup(reason=err)

        backup_service = self._map_service_to_driver(backup['service'])
        if backup_service is not None:
            configured_service = self.driver_name
            # TODO(tommylikehu): We upgraded the 'driver_name' from module
            # to class name, so we use 'in' here to match two namings,
            # this can be replaced with equal sign during next
            # release (Rocky).
            if backup_service not in configured_service:
                err = _('Delete backup aborted, the backup service currently'
                        ' configured [%(configured_service)s] is not the'
                        ' backup service that was used to create this'
                        ' backup [%(backup_service)s].')\
                    % {'configured_service': configured_service,
                       'backup_service': backup_service}
                self._update_backup_error(backup, err)
                raise exception.InvalidBackup(reason=err)

            try:
                backup_service = self.get_backup_driver(context)
                backup_service.delete_backup(backup)
            except Exception as err:
                with excutils.save_and_reraise_exception():
                    self._update_backup_error(backup, six.text_type(err))

        # Get reservations
        try:
            reserve_opts = {
                'backups': -1,
                'backup_gigabytes': -backup.size,
            }
            reservations = QUOTAS.reserve(context,
                                          project_id=backup.project_id,
                                          **reserve_opts)
        except Exception:
            reservations = None
            LOG.exception("Failed to update usages deleting backup")

        if backup.encryption_key_id is not None:
            volume_utils.delete_encryption_key(context,
                                               key_manager.API(CONF),
                                               backup.encryption_key_id)
            backup.encryption_key_id = None
            backup.save()

        backup.destroy()
        # If this backup is incremental backup, handle the
        # num_dependent_backups of parent backup
        if backup.parent_id:
            parent_backup = objects.Backup.get_by_id(context,
                                                     backup.parent_id)
            if parent_backup.has_dependent_backups:
                parent_backup.num_dependent_backups -= 1
                parent_backup.save()
        # Commit the reservations
        if reservations:
            QUOTAS.commit(context, reservations,
                          project_id=backup.project_id)

        LOG.info('Delete backup finished, backup %s deleted.', backup.id)
        self._notify_about_backup_usage(context, backup, "delete.end")

    def _notify_about_backup_usage(self,
                                   context,
                                   backup,
                                   event_suffix,
                                   extra_usage_info=None):
        volume_utils.notify_about_backup_usage(
            context, backup, event_suffix,
            extra_usage_info=extra_usage_info,
            host=self.host)

    def export_record(self, context, backup):
        """Export all volume backup metadata details to allow clean import.

        Export backup metadata so it could be re-imported into the database
        without any prerequisite in the backup database.

        :param context: running context
        :param backup: backup object to export
        :returns: backup_record - a description of how to import the backup
        :returns: contains 'backup_url' - how to import the backup, and
        :returns: 'backup_service' describing the needed driver.
        :raises InvalidBackup:
        """
        LOG.info('Export record started, backup: %s.', backup.id)

        expected_status = fields.BackupStatus.AVAILABLE
        actual_status = backup.status
        if actual_status != expected_status:
            err = (_('Export backup aborted, expected backup status '
                     '%(expected_status)s but got %(actual_status)s.') %
                   {'expected_status': expected_status,
                    'actual_status': actual_status})
            raise exception.InvalidBackup(reason=err)

        backup_record = {'backup_service': backup.service}
        backup_service = self._map_service_to_driver(backup.service)
        configured_service = self.driver_name
        # TODO(tommylikehu): We upgraded the 'driver_name' from module
        # to class name, so we use 'in' here to match two namings,
        # this can be replaced with equal sign during next
        # release (Rocky).
        if backup_service not in configured_service:
            err = (_('Export record aborted, the backup service currently '
                     'configured [%(configured_service)s] is not the '
                     'backup service that was used to create this '
                     'backup [%(backup_service)s].') %
                   {'configured_service': configured_service,
                    'backup_service': backup_service})
            raise exception.InvalidBackup(reason=err)

        # Call driver to create backup description string
        try:
            backup_service = self.get_backup_driver(context)
            driver_info = backup_service.export_record(backup)
            backup_url = backup.encode_record(driver_info=driver_info)
            backup_record['backup_url'] = backup_url
        except Exception as err:
            msg = six.text_type(err)
            raise exception.InvalidBackup(reason=msg)

        LOG.info('Export record finished, backup %s exported.', backup.id)
        return backup_record

    def import_record(self,
                      context,
                      backup,
                      backup_service,
                      backup_url,
                      backup_hosts):
        """Import all volume backup metadata details to the backup db.

        :param context: running context
        :param backup: The new backup object for the import
        :param backup_service: The needed backup driver for import
        :param backup_url: An identifier string to locate the backup
        :param backup_hosts: Potential hosts to execute the import
        :raises InvalidBackup:
        :raises ServiceNotFound:
        """
        LOG.info('Import record started, backup_url: %s.', backup_url)

        # Can we import this backup?
        if backup_service != self.driver_name:
            # No, are there additional potential backup hosts in the list?
            if len(backup_hosts) > 0:
                # try the next host on the list, maybe he can import
                first_host = backup_hosts.pop()
                self.backup_rpcapi.import_record(context,
                                                 first_host,
                                                 backup,
                                                 backup_service,
                                                 backup_url,
                                                 backup_hosts)
            else:
                # empty list - we are the last host on the list, fail
                err = _('Import record failed, cannot find backup '
                        'service to perform the import. Request service '
                        '%(service)s.') % {'service': backup_service}
                self._update_backup_error(backup, err)
                raise exception.ServiceNotFound(service_id=backup_service)
        else:
            # Yes...
            try:
                # Deserialize backup record information
                backup_options = backup.decode_record(backup_url)

                # Extract driver specific info and pass it to the driver
                driver_options = backup_options.pop('driver_info', {})
                backup_service = self.get_backup_driver(context)
                backup_service.import_record(backup, driver_options)
            except Exception as err:
                msg = six.text_type(err)
                self._update_backup_error(backup, msg)
                raise exception.InvalidBackup(reason=msg)

            required_import_options = {
                'display_name',
                'display_description',
                'container',
                'size',
                'service_metadata',
                'object_count',
                'id'
            }

            # Check for missing fields in imported data
            missing_opts = required_import_options - set(backup_options)
            if missing_opts:
                msg = (_('Driver successfully decoded imported backup data, '
                         'but there are missing fields (%s).') %
                       ', '.join(missing_opts))
                self._update_backup_error(backup, msg)
                raise exception.InvalidBackup(reason=msg)

            # Confirm the ID from the record in the DB is the right one
            backup_id = backup_options['id']
            if backup_id != backup.id:
                msg = (_('Trying to import backup metadata from id %(meta_id)s'
                         ' into backup %(id)s.') %
                       {'meta_id': backup_id, 'id': backup.id})
                self._update_backup_error(backup, msg)
                raise exception.InvalidBackup(reason=msg)

            # Overwrite some fields
            backup_options['service'] = self.driver_name
            backup_options['availability_zone'] = self.az
            backup_options['host'] = self.host

            # Remove some values which are not actual fields and some that
            # were set by the API node
            for key in ('name', 'user_id', 'project_id', 'deleted_at',
                        'deleted', 'fail_reason', 'status'):
                backup_options.pop(key, None)

            # Update the database
            backup.update(backup_options)
            backup.save()

            # Verify backup
            try:
                if isinstance(backup_service, driver.BackupDriverWithVerify):
                    backup_service.verify(backup.id)
                else:
                    LOG.warning('Backup service %(service)s does not '
                                'support verify. Backup id %(id)s is '
                                'not verified. Skipping verify.',
                                {'service': self.driver_name,
                                 'id': backup.id})
            except exception.InvalidBackup as err:
                with excutils.save_and_reraise_exception():
                    self._update_backup_error(backup, six.text_type(err))

            # Update the backup's status
            backup.update({"status": fields.BackupStatus.AVAILABLE})
            backup.save()

            LOG.info('Import record id %s metadata from driver '
                     'finished.', backup.id)

    def reset_status(self, context, backup, status):
        """Reset volume backup status.

        :param context: running context
        :param backup: The backup object for reset status operation
        :param status: The status to be set
        :raises InvalidBackup:
        :raises BackupVerifyUnsupportedDriver:
        :raises AttributeError:
        """
        LOG.info('Reset backup status started, backup_id: '
                 '%(backup_id)s, status: %(status)s.',
                 {'backup_id': backup.id,
                  'status': status})

        backup_service_name = self._map_service_to_driver(backup.service)
        LOG.info('Backup service: %s.', backup_service_name)
        if backup_service_name is not None:
            configured_service = self.driver_name
            # TODO(tommylikehu): We upgraded the 'driver_name' from module
            # to class name, so we use 'in' here to match two namings,
            # this can be replaced with equal sign during next
            # release (Rocky).
            if backup_service_name not in configured_service:
                err = _('Reset backup status aborted, the backup service'
                        ' currently configured [%(configured_service)s] '
                        'is not the backup service that was used to create'
                        ' this backup [%(backup_service)s].') % \
                    {'configured_service': configured_service,
                     'backup_service': backup_service_name}
                raise exception.InvalidBackup(reason=err)
            # Verify backup
            try:
                # check whether the backup is ok or not
                if (status == fields.BackupStatus.AVAILABLE
                        and backup['status'] != fields.BackupStatus.RESTORING):
                    # check whether we could verify the backup is ok or not
                    backup_service = self.get_backup_driver(context)
                    if isinstance(backup_service,
                                  driver.BackupDriverWithVerify):
                        backup_service.verify(backup.id)
                        backup.status = status
                        backup.save()
                    # driver does not support verify function
                    else:
                        msg = (_('Backup service %(configured_service)s '
                                 'does not support verify. Backup id'
                                 ' %(id)s is not verified. '
                                 'Skipping verify.') %
                               {'configured_service': self.driver_name,
                                'id': backup.id})
                        raise exception.BackupVerifyUnsupportedDriver(
                            reason=msg)
                # reset status to error or from restoring to available
                else:
                    if (status == fields.BackupStatus.ERROR or
                        (status == fields.BackupStatus.AVAILABLE and
                            backup.status == fields.BackupStatus.RESTORING)):
                        backup.status = status
                        backup.save()
            except exception.InvalidBackup:
                with excutils.save_and_reraise_exception():
                    LOG.error("Backup id %s is not invalid. Skipping reset.",
                              backup.id)
            except exception.BackupVerifyUnsupportedDriver:
                with excutils.save_and_reraise_exception():
                    LOG.error('Backup service %(configured_service)s '
                              'does not support verify. Backup id '
                              '%(id)s is not verified. '
                              'Skipping verify.',
                              {'configured_service': self.driver_name,
                               'id': backup.id})
            except AttributeError:
                msg = (_('Backup service %(service)s does not support '
                         'verify. Backup id %(id)s is not verified. '
                         'Skipping reset.') %
                       {'service': self.driver_name,
                        'id': backup.id})
                LOG.error(msg)
                raise exception.BackupVerifyUnsupportedDriver(
                    reason=msg)

            # Needs to clean temporary volumes and snapshots.
            try:
                self._cleanup_temp_volumes_snapshots_for_one_backup(
                    context, backup)
            except Exception:
                LOG.exception("Problem cleaning temp volumes and "
                              "snapshots for backup %(bkup)s.",
                              {'bkup': backup.id})

            # send notification to ceilometer
            notifier_info = {'id': backup.id, 'update': {'status': status}}
            notifier = rpc.get_notifier('backupStatusUpdate')
            notifier.info(context, "backups.reset_status.end",
                          notifier_info)

    def check_support_to_force_delete(self, context):
        """Check if the backup driver supports force delete operation.

        :param context: running context
        """
        backup_service = self.get_backup_driver(context)
        return backup_service.support_force_delete

    def _attach_device(self, ctxt, backup_device,
                       properties, is_snapshot=False):
        """Attach backup device."""
        if not is_snapshot:
            return self._attach_volume(ctxt, backup_device, properties)
        else:
            return self._attach_snapshot(ctxt, backup_device, properties)

    def _attach_volume(self, context, volume, properties):
        """Attach a volume."""

        try:
            conn = self.volume_rpcapi.initialize_connection(context,
                                                            volume,
                                                            properties)
            return self._connect_device(conn)
        except Exception:
            with excutils.save_and_reraise_exception():
                try:
                    self.volume_rpcapi.terminate_connection(context, volume,
                                                            properties,
                                                            force=True)
                except Exception:
                    LOG.warning("Failed to terminate the connection "
                                "of volume %(volume_id)s, but it is "
                                "acceptable.",
                                {'volume_id', volume.id})

    def _attach_snapshot(self, ctxt, snapshot, properties):
        """Attach a snapshot."""

        try:
            conn = self.volume_rpcapi.initialize_connection_snapshot(
                ctxt, snapshot, properties)
            return self._connect_device(conn)
        except Exception:
            with excutils.save_and_reraise_exception():
                try:
                    self.volume_rpcapi.terminate_connection_snapshot(
                        ctxt, snapshot, properties, force=True)
                except Exception:
                    LOG.warning("Failed to terminate the connection "
                                "of snapshot %(snapshot_id)s, but it is "
                                "acceptable.",
                                {'snapshot_id', snapshot.id})

    def _connect_device(self, conn):
        """Establish connection to device."""
        use_multipath = CONF.use_multipath_for_image_xfer
        device_scan_attempts = CONF.num_volume_device_scan_tries
        protocol = conn['driver_volume_type']
        connector = utils.brick_get_connector(
            protocol,
            use_multipath=use_multipath,
            device_scan_attempts=device_scan_attempts,
            conn=conn)
        vol_handle = connector.connect_volume(conn['data'])

        return {'conn': conn, 'device': vol_handle, 'connector': connector}

    def _detach_device(self, ctxt, attach_info, device,
                       properties, is_snapshot=False, force=False,
                       ignore_errors=False):
        """Disconnect the volume or snapshot from the host. """
        connector = attach_info['connector']
        connector.disconnect_volume(attach_info['conn']['data'],
                                    attach_info['device'],
                                    force=force, ignore_errors=ignore_errors)
        rpcapi = self.volume_rpcapi
        if not is_snapshot:
            rpcapi.terminate_connection(ctxt, device, properties,
                                        force=force)
            rpcapi.remove_export(ctxt, device)
        else:
            rpcapi.terminate_connection_snapshot(ctxt, device,
                                                 properties, force=force)
            rpcapi.remove_export_snapshot(ctxt, device)

    def is_working(self):
        return self.is_initialized

    @periodic_task.periodic_task(spacing=CONF.periodic_interval)
    def _report_driver_status(self, context):
        if not self.is_working():
            self.setup_backup_backend(context)
