Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add support for using registries.json to handle public/private useren… #568

Merged
merged 3 commits into from
Nov 21, 2024
Merged

add support for using registries.json to handle public/private useren… #568

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
962494f
add support for using registries.json to handle public/private useren…
k-rister Nov 18, 2024
ff3f130
rework how repo url decoding is handled so that it doesn't matter wha…
k-rister Nov 20, 2024
debe757
add the ability for rickshaw-run to search for "external" userenv jso…
k-rister Nov 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 37 additions & 11 deletions endpoints/remotehosts/remotehosts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ def init_settings():
settings["misc"]["image-map"] = dict()
images = args.images.split(",")
for image in images:
image_split = image.split("::")
image_split = image.split("::", maxsplit=1)
settings["misc"]["image-map"][image_split[0]] = image_split[1]

log_settings(mode = "misc")
Expand Down Expand Up @@ -725,19 +725,34 @@ def image_pull_worker_thread(thread_id, work_queue, threads_rcs):
thread_logger(thread_name, "Remote user is %s" % (my_run_file_remote["config"]["settings"]["remote-user"]), remote_name = remote)

with endpoints.remote_connection(remote, my_run_file_remote["config"]["settings"]["remote-user"]) as c:
for image in my_unique_remote["images"]:
result = endpoints.run_remote(c, "podman pull " + image)
for image_info in my_unique_remote["images"]:
auth_arg = ""
remote_auth_file = settings["dirs"]["remote"]["run"] + "/pull-token.json"
if "pull-token" in image_info:
thread_logger(thread_name, "Image %s requires pull token %s" % (image_info["image"], image_info["pull-token"]), remote_name = remote)
result = c.put(image_info["pull-token"], remote_auth_file)
thread_logger(thread_name, "Copied %s to %s:%s" % (image_info["pull-token"], remote, remote_auth_file), remote_name = remote)
auth_arg = "--authfile=" + remote_auth_file

result = endpoints.run_remote(c, "podman pull " + auth_arg + " " + image_info["image"])
loglevel = "info"
if result.exited != 0:
loglevel = "error"
thread_logger(thread_name, "Attempted to pull %s with return code %d:\nstdout:\n%sstderr:\n%s" % (image, result.exited, result.stdout, result.stderr), log_level = loglevel, remote_name = remote)
thread_logger(thread_name, "Attempted to pull %s with return code %d:\nstdout:\n%sstderr:\n%s" % (image_info["image"], result.exited, result.stdout, result.stderr), log_level = loglevel, remote_name = remote)
rc += result.exited

result = endpoints.run_remote(c, "echo '" + image + " " + str(int(time.time())) + " " + args.run_id + "' >> " + settings["dirs"]["remote"]["base"] + "/remotehosts-container-image-census")
if "pull-token" in image_info:
result = endpoints.run_remote(c, "rm -v " + remote_auth_file)
log_level = "info"
if result.exited != 0:
loglevel = "error"
thread_logger(thread_name, "Attempted to remove %s with return code %d:\nstdout:\n%sstderr:\n%s" % (remote_auth_file, result.exited, result.stdout, result.stderr), log_level = loglevel, remote_name = remote)

result = endpoints.run_remote(c, "echo '" + image_info["image"] + " " + str(int(time.time())) + " " + args.run_id + "' >> " + settings["dirs"]["remote"]["base"] + "/remotehosts-container-image-census")
loglevel = "info"
if result.exited != 0:
loglevel = "error"
thread_logger(thread_name, "Recorded usage for %s in the census with return code %d:\nstdout:\n%sstderr:\n%s" % (image, result.exited, result.stdout, result.stderr), log_level = loglevel, remote_name = remote)
thread_logger(thread_name, "Recorded usage for %s in the census with return code %d:\nstdout:\n%sstderr:\n%s" % (image_info["image"], result.exited, result.stdout, result.stderr), log_level = loglevel, remote_name = remote)
rc += result.exited

thread_logger(thread_name, "Notifying work queue that job processing is complete", remote_name = remote)
Expand Down Expand Up @@ -894,18 +909,27 @@ def remotes_pull_images():
"""
log.info("Determining which images to pull to which remotes")
for remote in settings["engines"]["remotes"].keys():
if not "images" in settings["engines"]["remotes"][remote]:
settings["engines"]["remotes"][remote]["images"] = []
if not "raw-images" in settings["engines"]["remotes"][remote]:
settings["engines"]["remotes"][remote]["raw-images"] = []

for role in settings["engines"]["remotes"][remote]["roles"].keys():
for id in settings["engines"]["remotes"][remote]["roles"][role]["ids"]:
image = get_engine_id_image(role, id)
if image is None:
log.error("Could not find image for remote %s with role %s and id %d" % (remote, role, str(id)))
else:
settings["engines"]["remotes"][remote]["images"].append(image)

settings["engines"]["remotes"][remote]["images"] = list(set(settings["engines"]["remotes"][remote]["images"]))
settings["engines"]["remotes"][remote]["raw-images"].append(image)

settings["engines"]["remotes"][remote]["raw-images"] = list(set(settings["engines"]["remotes"][remote]["raw-images"]))
settings["engines"]["remotes"][remote]["images"] = []
for image in settings["engines"]["remotes"][remote]["raw-images"]:
image_info = dict()
image_split = image.split("::")
image_info["image"] = image_split[0]
if len(image_split) > 1:
log.info("Found image %s with pull token %s" % (image_split[0], image_split[1]))
image_info["pull-token"] = image_split[1]
settings["engines"]["remotes"][remote]["images"].append(image_info)

log_settings(mode = "engines")

Expand Down Expand Up @@ -1434,6 +1458,8 @@ def launch_engines_worker_thread(thread_id, work_queue, threads_rcs):
thread_logger(thread_name, "Could not determine image", log_level = "error", remote_name = remote_name, engine_name = engine_name)
continue
else:
image_split = image.split("::")
image = image_split[0]
thread_logger(thread_name, "Image is '%s'" % (image), remote_name = remote_name, engine_name = engine_name)

osruntime = None
Expand Down
Loading
Loading