Skip to content

Commit

Permalink
Reuse OSD ID in migrations, support replace.osds
Browse files Browse the repository at this point in the history 
These functions are essentially equivalent and bundled together.

Signed-off-by: Eric Jackson <[email protected]>
  • Loading branch information
@swiftgist
swiftgist committed Jul 6, 2018
1 parent 86a614d commit 3b21e44
Show file tree
Hide file tree
Showing 71 changed files with 4,076 additions and 154 deletions.
33 changes: 31 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,12 @@ copy-files:
install -d -m 755 $(DESTDIR)/srv/salt/ceph/tests/quiescent/timeout
install -m 644 srv/salt/ceph/tests/quiescent/*.sls $(DESTDIR)/srv/salt/ceph/tests/quiescent
install -m 644 srv/salt/ceph/tests/quiescent/timeout/*.sls $(DESTDIR)/srv/salt/ceph/tests/quiescent/timeout
install -d -m 755 $(DESTDIR)/srv/salt/ceph/tests/migrate
install -m 644 srv/salt/ceph/tests/migrate/*.sls $(DESTDIR)/srv/salt/ceph/tests/migrate
install -d -m 755 $(DESTDIR)/srv/salt/ceph/tests/remove
install -m 644 srv/salt/ceph/tests/remove/*.sls $(DESTDIR)/srv/salt/ceph/tests/remove
install -d -m 755 $(DESTDIR)/srv/salt/ceph/tests/replace
install -m 644 srv/salt/ceph/tests/replace/*.sls $(DESTDIR)/srv/salt/ceph/tests/replace
install -d -m 755 $(DESTDIR)/srv/salt/ceph/tests/restart
install -d -m 755 $(DESTDIR)/srv/salt/ceph/tests/restart/mon
install -d -m 755 $(DESTDIR)/srv/salt/ceph/tests/restart/mon/change
Expand Down Expand Up @@ -130,6 +136,22 @@ copy-files:
install -m 644 srv/salt/ceph/smoketests/macros/os_switch/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/macros/os_switch
install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/quiescent
install -m 644 srv/salt/ceph/smoketests/quiescent/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/quiescent
install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/migrate
install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/migrate/filestore
install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/migrate/filestore2
install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/migrate/bluestore
install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/migrate/bluestore2
install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/migrate/bluestore3
install -m 644 srv/salt/ceph/smoketests/migrate/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/migrate
install -m 644 srv/salt/ceph/smoketests/migrate/filestore/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/migrate/filestore
install -m 644 srv/salt/ceph/smoketests/migrate/filestore2/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/migrate/filestore2
install -m 644 srv/salt/ceph/smoketests/migrate/bluestore/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/migrate/bluestore
install -m 644 srv/salt/ceph/smoketests/migrate/bluestore2/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/migrate/bluestore2
install -m 644 srv/salt/ceph/smoketests/migrate/bluestore3/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/migrate/bluestore3
install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/replace
install -m 644 srv/salt/ceph/smoketests/replace/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/replace
install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/remove
install -m 644 srv/salt/ceph/smoketests/remove/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/remove
install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/restart
install -m 644 srv/salt/ceph/smoketests/restart/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/restart
# docs
Expand Down Expand Up @@ -181,6 +203,11 @@ copy-files:
# modules
install -d -m 755 $(DESTDIR)/srv/salt/_modules
install -m 644 srv/salt/_modules/*.py* $(DESTDIR)/srv/salt/_modules/
# state modules
install -d -m 755 $(DESTDIR)/srv/salt/_states
install -d -m 755 $(DESTDIR)/srv/salt/_states/__pycache__
install -m 644 srv/salt/_states/__pycache__/*.pyc $(DESTDIR)/srv/salt/_states/__pycache__
install -m 644 srv/salt/_states/*.py* $(DESTDIR)/srv/salt/_states/
# state files
install -d -m 755 $(DESTDIR)/srv/salt/ceph/admin
install -m 644 srv/salt/ceph/admin/*.sls $(DESTDIR)/srv/salt/ceph/admin/
Expand Down Expand Up @@ -510,8 +537,10 @@ copy-files:
install -m 644 srv/salt/ceph/remove/igw/auth/*.sls $(DESTDIR)/srv/salt/ceph/remove/igw/auth/
install -d -m 755 $(DESTDIR)/srv/salt/ceph/remove/mds
install -m 644 srv/salt/ceph/remove/mds/*.sls $(DESTDIR)/srv/salt/ceph/remove/mds/
install -d -m 755 $(DESTDIR)/srv/salt/ceph/remove/migrated
install -m 644 srv/salt/ceph/remove/migrated/*.sls $(DESTDIR)/srv/salt/ceph/remove/migrated/
install -d -m 755 $(DESTDIR)/srv/salt/ceph/remove/destroyed
install -m 644 srv/salt/ceph/remove/destroyed/*.sls $(DESTDIR)/srv/salt/ceph/remove/destroyed/
# Renamed for deprecation
ln -sf destroyed $(DESTDIR)/srv/salt/ceph/remove/migrated
install -d -m 755 $(DESTDIR)/srv/salt/ceph/remove/mgr
install -m 644 srv/salt/ceph/remove/mgr/*.sls $(DESTDIR)/srv/salt/ceph/remove/mgr/
install -d -m 755 $(DESTDIR)/srv/salt/ceph/remove/mon
Expand Down
33 changes: 31 additions & 2 deletions deepsea.spec.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ systemctl try-restart salt-api > /dev/null 2>&1 || :
%dir %attr(0755, salt, salt) /srv/pillar/ceph/benchmarks/fio
%dir %attr(0755, salt, salt) /srv/pillar/ceph/benchmarks/templates
%dir /srv/salt/_modules
%dir /srv/salt/_states
%dir /srv/salt/_states/__pycache__
%dir /srv/modules
%dir /srv/modules/modules
%dir /srv/modules/runners
Expand Down Expand Up @@ -238,13 +240,13 @@ systemctl try-restart salt-api > /dev/null 2>&1 || :
%dir /srv/salt/ceph/redeploy/osds
%dir /srv/salt/ceph/redeploy/nodes
%dir /srv/salt/ceph/remove
%dir /srv/salt/ceph/remove/destroyed
%dir /srv/salt/ceph/remove/ganesha
%dir /srv/salt/ceph/remove/igw
%dir /srv/salt/ceph/remove/igw/auth
%dir /srv/salt/ceph/remove/mon
%dir /srv/salt/ceph/remove/mds
%dir /srv/salt/ceph/remove/mgr
%dir /srv/salt/ceph/remove/migrated
%dir /srv/salt/ceph/remove/rgw
%dir /srv/salt/ceph/remove/storage
%dir /srv/salt/ceph/remove/storage/drain
Expand Down Expand Up @@ -395,6 +397,7 @@ systemctl try-restart salt-api > /dev/null 2>&1 || :
%config(noreplace) %attr(-, salt, salt) /etc/salt/master.d/*.conf
/srv/modules/modules/*.py*
/srv/modules/runners/*.py*
%exclude /srv/modules/runners/smoketests.py
/srv/modules/utils/*.py*
%config %attr(-, salt, salt) /srv/pillar/top.sls
%config %attr(-, salt, salt) /srv/pillar/ceph/init.sls
Expand All @@ -406,6 +409,8 @@ systemctl try-restart salt-api > /dev/null 2>&1 || :
%config(noreplace) %attr(-, salt, salt) /srv/pillar/ceph/deepsea_minions.sls
%config %attr(-, salt, salt) /srv/pillar/ceph/stack/stack.cfg
/srv/salt/_modules/*.py*
/srv/salt/_states/*.py*
/srv/salt/_states/__pycache__/*.py*
%config /srv/salt/ceph/admin/*.sls
%config /srv/salt/ceph/admin/files/*.j2
%config /srv/salt/ceph/admin/key/*.sls
Expand Down Expand Up @@ -550,7 +555,8 @@ systemctl try-restart salt-api > /dev/null 2>&1 || :
%config /srv/salt/ceph/remove/igw/auth/*.sls
%config /srv/salt/ceph/remove/mon/*.sls
%config /srv/salt/ceph/remove/mds/*.sls
%config /srv/salt/ceph/remove/migrated/*.sls
%config /srv/salt/ceph/remove/destroyed/*.sls
%config /srv/salt/ceph/remove/migrated
%config /srv/salt/ceph/remove/mgr/*.sls
%config /srv/salt/ceph/remove/openattic/*.sls
%config /srv/salt/ceph/remove/rgw/*.sls
Expand Down Expand Up @@ -729,6 +735,14 @@ the README for more information.
%dir /srv/salt/ceph/smoketests/macros
%dir /srv/salt/ceph/smoketests/macros/os_switch
%dir /srv/salt/ceph/smoketests/quiescent
%dir /srv/salt/ceph/smoketests/migrate
%dir /srv/salt/ceph/smoketests/migrate/filestore
%dir /srv/salt/ceph/smoketests/migrate/filestore2
%dir /srv/salt/ceph/smoketests/migrate/bluestore
%dir /srv/salt/ceph/smoketests/migrate/bluestore2
%dir /srv/salt/ceph/smoketests/migrate/bluestore3
%dir /srv/salt/ceph/smoketests/remove
%dir /srv/salt/ceph/smoketests/replace
%dir /srv/salt/ceph/smoketests/restart
%dir /srv/salt/ceph/tests
%dir /srv/salt/ceph/tests/keyrings
Expand All @@ -737,6 +751,9 @@ the README for more information.
%dir /srv/salt/ceph/tests/quiescent/timeout
%dir /srv/salt/ceph/tests/quiescent
%dir /srv/salt/ceph/tests/quiescent/timeout
%dir /srv/salt/ceph/tests/migrate
%dir /srv/salt/ceph/tests/remove
%dir /srv/salt/ceph/tests/replace
%dir /srv/salt/ceph/tests/restart
%dir /srv/salt/ceph/tests/restart/mon
%dir /srv/salt/ceph/tests/restart/mon/change
Expand All @@ -755,14 +772,26 @@ the README for more information.
%dir /srv/salt/ceph/tests/restart/rgw/forced
%dir /srv/salt/ceph/tests/restart/rgw/nochange
/srv/salt/ceph/smoketests/apparmor/*.sls
/srv/modules/runners/smoketests.py
/srv/salt/ceph/smoketests/keyrings/*.sls
/srv/salt/ceph/smoketests/macros/os_switch/*.sls
/srv/salt/ceph/smoketests/quiescent/*.sls
/srv/salt/ceph/smoketests/migrate/*.sls
/srv/salt/ceph/smoketests/migrate/filestore/*.sls
/srv/salt/ceph/smoketests/migrate/filestore2/*.sls
/srv/salt/ceph/smoketests/migrate/bluestore/*.sls
/srv/salt/ceph/smoketests/migrate/bluestore2/*.sls
/srv/salt/ceph/smoketests/migrate/bluestore3/*.sls
/srv/salt/ceph/smoketests/remove/*.sls
/srv/salt/ceph/smoketests/replace/*.sls
/srv/salt/ceph/smoketests/restart/*.sls
/srv/salt/ceph/tests/keyrings/*.sls
/srv/salt/ceph/tests/os_switch/*.sls
/srv/salt/ceph/tests/quiescent/*.sls
/srv/salt/ceph/tests/quiescent/timeout/*.sls
/srv/salt/ceph/tests/migrate/*.sls
/srv/salt/ceph/tests/remove/*.sls
/srv/salt/ceph/tests/replace/*.sls
/srv/salt/ceph/tests/restart/*.sls
/srv/salt/ceph/tests/restart/mon/*.sls
/srv/salt/ceph/tests/restart/mon/change/*.sls
Expand Down
60 changes: 60 additions & 0 deletions srv/modules/runners/advise.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,3 +102,63 @@ def networks():
__func_alias__ = {
'help_': 'help',
}


def osds():
"""
Inform the admin of pending changes and appropriate actions
Note: I went with the mapping here such as 'unconfigured' implies
'deploy'. This is more about communicating with the maintainers
although picking the "best" name and propogating may be a solution.
The deploy and redeploy are osd methods.
"""
local = salt.client.LocalClient()
report = local.cmd('I@roles:storage', 'osd.report',
['human=False'], tgt_type="compound")

bold = '\033[1m'
endc = '\033[0m'

unconfigured = _tidy('unconfigured', report)
changed = _tidy('changed', report)
unmounted = _tidy('unmounted', report)

messages = {'deploy': {'header': '\nThese devices will be deployed',
'footer': "Run 'salt-run state.orch ceph.stage.3'"},
'redeploy': {'header': "\nThe devices will be redeployed",
'footer': "Run 'salt-run state.orch ceph.migrate.osds'"},
'stale': {'header': "\nVerify that these devices are in the desired state",
'footer': "Run 'salt MINION osd.delete_grain ID' for a stale entry"}}

if unconfigured:
print(messages['deploy']['header'])
print("{}{}{}".format(bold, unconfigured, endc))
print(messages['deploy']['footer'])

if changed:
print(messages['redeploy']['header'])
print("{}{}{}".format(bold, changed, endc))
print(messages['redeploy']['footer'])

if unmounted:
print(messages['stale']['header'])
print("{}{}{}".format(bold, unmounted, endc))
print(messages['stale']['footer'])

return ""


def _tidy(key, report):
"""
Return a line of minion followed by comma separated devices if present
"""
line = ""
for minion in sorted(report):
if report[minion][key]:
if len(minion) + len(", ".join(report[minion][key])) < 80:
line += "{}: {}\n".format(minion, ", ".join(sorted(report[minion][key])))
else:
line += "\n{}:\n {}\n".format(minion, "\n ".join(sorted(report[minion][key])))
return line
12 changes: 12 additions & 0 deletions srv/modules/runners/push.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,18 @@ def proposal(filename="/srv/pillar/ceph/proposals/policy.cfg", dryrun=False):
return True


def organize(filename="/srv/pillar/ceph/proposals/policy.cfg"):
"""
Read the passed filename, organize the files with common subdirectories
"""
if not os.path.isfile(filename):
log.warning("{} is missing".format(filename))
return ""
pillar_data = PillarData()
common = pillar_data.organize(filename)
return common


def convert(filename="/srv/pillar/ceph/proposals/policy.cfg"):
"""
Convert the hardware profiles that policy.cfg is using and update
Expand Down
71 changes: 19 additions & 52 deletions srv/modules/runners/remove.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,72 +18,39 @@ def help_():
"""
Usage
"""
usage = ('salt-run remove.osd id:\n\n'
usage = ('salt-run remove.osd id [id ...][force=True]:\n\n'
' Removes an OSD\n'
'\n\n')
print(usage)
return ""


def osd(id_, drain=False):
def osd(*args, **kwargs):
"""
Removes an OSD gracefully
Remove an OSD gracefully or forcefully. Always attempt to remove
ID from Ceph even if OSD has been removed from the minion.
"""
runner_cli = salt.runner.RunnerClient(
salt.config.client_config('/etc/salt/master'))
result = __salt__['replace.osd'](*args, called=True, **kwargs)

if not runner_cli.cmd('disengage.check'):
log.error(('Safety is not disengaged...refusing to remove OSD',
' run "salt-run disengage.safety" first'
' THIS WILL CAUSE DATA LOSS.'))
return False
# Replace OSD exited early
if not result:
return ""

if id_ < 0:
log.error('Bogus id supplied...OSDs have IDs >= 0')
return False
master_minion = result['master_minion']
osds = result['osds']

local_cli = salt.client.LocalClient()
local = salt.client.LocalClient()

osds = local_cli.cmd('I@roles:storage', 'osd.list', tgt_type='compound')
for osd_id in osds:
cmds = ['ceph osd crush remove osd.{}'.format(osd_id),
'ceph auth del osd.{}'.format(osd_id),
'ceph osd rm {}'.format(osd_id)]

host = ''
for _osd in osds:
if '{}'.format(id_) in osds[_osd]:
host = _osd
break
else:
log.error('No OSD with ID {} found...giving up'.format(id_))
return False
print("Removing osd {} from Ceph".format(osd_id))
for cmd in cmds:
local.cmd(master_minion, 'cmd.run', [cmd], tgt_type='compound')

master_minion = list(local_cli.cmd('I@roles:master', 'pillar.get',
['master_minion'],
tgt_type='compound').items())[0][1]

if drain:
log.info('Draining OSD {} now'.format(id_))
ret = local_cli.cmd(host, 'osd.zero_weight', [id_])

log.info('Setting OSD {} out'.format(id_))

ret = local_cli.cmd(master_minion, 'cmd.run',
['ceph osd out {}'.format(id_)])

log.info('Stopping and wiping OSD {} now'.format(id_))

ret = local_cli.cmd(host, 'osd.remove', [id_])
log.info(ret)

ret = local_cli.cmd(master_minion, 'cmd.run',
['ceph osd crush remove osd.{}'.format(id_)])
log.info(ret)
ret = local_cli.cmd(master_minion, 'cmd.run',
['ceph auth del osd.{}'.format(id_)])
log.info(ret)
ret = local_cli.cmd(master_minion, 'cmd.run',
['ceph osd rm {}'.format(id_)])
log.info(ret)

return True
return ""

__func_alias__ = {
'help_': 'help',
Expand Down
Loading

0 comments on commit 3b21e44

Please sign in to comment.