From 2f824798579442009267340fc7873196ef176ec1 Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Fri, 3 Jul 2026 16:09:49 +0200 Subject: [PATCH 1/9] add option in compute worker env to not send logs to the instance, instead writing them in a local file --- compute_worker/compute_worker.py | 163 +++++++++++++++++++------------ 1 file changed, 102 insertions(+), 61 deletions(-) diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index 2b388187f..5d22915dc 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -121,6 +121,9 @@ def to_bool(val): ) COMPETITION_ALLOW_IMAGE_PULL = to_bool(get("COMPETITION_ALLOW_IMAGE_PULL", "True")) + SILENT_COMPUTE_WORKER = get("SILENT_COMPUTE_WORKER", "false").lower() + + # ----------------------------------------------- # Program Kind @@ -373,10 +376,15 @@ def run_wrapper(run_args): except SubmissionException as e: msg = str(e).strip() if msg: - msg = f"Submission failed: {msg}. See logs for more details." + if Settings.SILENT_COMPUTE_WORKER == "true": + msg = f"Submission failed: {msg}. Contact the Organizer for more details." + else: + msg = f"Submission failed: {msg}. See logs for more details." else: - msg = "Submission failed. See logs for more details." - + if Settings.SILENT_COMPUTE_WORKER == "true": + msg = "Submission failed. Contact the Organizer for more details." + else: + msg = "Submission failed. See logs for more details." run._update_status(SubmissionStatus.FAILED, extra_information=msg) raise @@ -602,18 +610,36 @@ async def watch_detailed_results(self): def push_logs(self): """Upload any collected logs, even in case of crash. """ - try: - for kind, logs in (self.logs or {}).items(): - for stream_key in ("stdout", "stderr"): - entry = logs.get(stream_key) if isinstance(logs, dict) else None - if not entry: - continue - location = entry.get("location") - data = entry.get("data") or b"" - if location: - self._put_file(location, raw_data=data) - except Exception as e: - logger.exception(f"Failed best-effort log upload: {e}") + if Settings.SILENT_COMPUTE_WORKER == "false": + try: + for kind, logs in (self.logs or {}).items(): + for stream_key in ("stdout", "stderr"): + entry = logs.get(stream_key) if isinstance(logs, dict) else None + if not entry: + continue + location = entry.get("location") + data = entry.get("data") or b"" + if location: + self._put_file(location, raw_data=data) + except Exception as e: + logger.exception(f"Failed best-effort log upload: {e}") + + else: + try: + logs_path = os.path.join(self.root_dir, "logs") + with open(logs_path, "w") as f: + for kind, logs in (self.logs or {}).items(): + for stream_key in ("stdout", "stderr"): + entry = logs.get(stream_key) if isinstance(logs, dict) else None + if not entry: + continue + location = entry.get("location") + data = entry.get("data") or b"" + if location: + f.write(str(data)) + except Exception as e: + logger.exception(f"Failed best-effort log file creation: {e}") + def get_detailed_results_file_path(self): default_detailed_results_path = os.path.join( @@ -769,12 +795,17 @@ def _get_container_image(self, image_name): self._update_submission(docker_pull_fail_data) # Send error through web socket to the frontend asyncio.run(self._send_data_through_socket(str(pull_error))) - raise DockerImagePullException( - f"Pull for {image_name} failed! Check the logs for more information" - ) + if Settings.SILENT_COMPUTE_WORKER == "true": + raise DockerImagePullException( + f"Pull for {image_name} failed! Contact the Organizer for more details." + ) + else: + raise DockerImagePullException( + f"Pull for {image_name} failed! Check the logs for more information" + ) else: logger.warning("Failed. Retrying in 5 seconds...") - time.sleep(5) # Wait 5 seconds before retrying + time.sleep(5) # Wait 5 seconds before retrying else: logger.info("COMPETITION_ALLOW_IMAGE_PULL is set to False, using local image if it exists") try: @@ -1004,21 +1035,24 @@ async def _run_container_engine_cmd(self, container, kind): # Create a websocket to send the logs in real time to the codabench instance # We need to set a timeout for the websocket connection otherwise the program will get stuck if he websocket does not connect. websocket = None - try: - websocket_url = f"{self.websocket_url}?kind={kind}" - logger.debug(f"Connecting to {websocket_url} for container {str(container.get('Id'))}") - websocket = await asyncio.wait_for( - websockets.connect(websocket_url), timeout=10.0 - ) - logger.debug(f"connected to {websocket_url} for container {str(container.get('Id'))}") - except Exception as e: - logger.error( - f"There was an error trying to connect to the websocket on the codabench instance: {e}" - ) + # Do not create a websocket if the real time logs are not wanted (Silent Compute Worker) + if Settings.SILENT_COMPUTE_WORKER == "false": + try: + websocket_url = f"{self.websocket_url}?kind={kind}" + logger.debug(f"Connecting to {websocket_url} for container {str(container.get('Id'))}") + websocket = await asyncio.wait_for( + websockets.connect(websocket_url), timeout=10.0 + ) + logger.debug(f"connected to {websocket_url} for container {str(container.get('Id'))}") - if Settings.LOG_LEVEL == Settings.LOG_LEVEL_DEBUG: - logger.exception(e) + except Exception as e: + logger.error( + f"There was an error trying to connect to the websocket on the codabench instance: {e}" + ) + + if Settings.LOG_LEVEL == Settings.LOG_LEVEL_DEBUG: + logger.exception(e) start = time.time() @@ -1034,6 +1068,7 @@ async def _run_container_engine_cmd(self, container, kind): ) # If we enter the for loop after the container exited, the program will get stuck + # Do not send the real time logs if they are not wanted (Silent Compute Worker) if client.inspect_container(container)["State"]["Status"].lower() == "running": logger.debug( "Show the logs and stream them to codabench " + container.get("Id") @@ -1043,25 +1078,27 @@ async def _run_container_engine_cmd(self, container, kind): if log[0] is not None: stdout_chunks.append(log[0]) logger.info(log[0].decode()) - try: - if websocket is not None: - await websocket.send( - json.dumps({"kind": kind, "message": log[0].decode()}) - ) - except Exception as e: - logger.error(e) + if Settings.SILENT_COMPUTE_WORKER == "false": + try: + if websocket is not None: + await websocket.send( + json.dumps({"kind": kind, "message": log[0].decode()}) + ) + except Exception as e: + logger.error(e) # Errors elif log[1] is not None: stderr_chunks.append(log[1]) logger.error(log[1].decode()) - try: - if websocket is not None: - await websocket.send( - json.dumps({"kind": kind, "message": log[1].decode()}) - ) - except Exception as e: - logger.error(e) + if Settings.SILENT_COMPUTE_WORKER == "false": + try: + if websocket is not None: + await websocket.send( + json.dumps({"kind": kind, "message": log[1].decode()}) + ) + except Exception as e: + logger.error(e) except (docker.errors.NotFound, docker.errors.APIError) as e: logger.error(e) @@ -1077,15 +1114,17 @@ async def _run_container_engine_cmd(self, container, kind): # Gets the logs of the container, sperating stdout and stderr (first and second position) thanks for demux=True return_Code = client.wait(container) logs_Unified = (b"".join(stdout_chunks), b"".join(stderr_chunks)) - logger.debug( - f"WORKER_MARKER: Disconnecting from {websocket_url}, program counter = {self.completed_program_counter}" - ) - if websocket is not None: - try: - await websocket.close() - await websocket.wait_closed() - except Exception as e: - logger.error(e) + + if Settings.SILENT_COMPUTE_WORKER == "false": + logger.debug( + f"WORKER_MARKER: Disconnecting from {websocket_url}, program counter = {self.completed_program_counter}" + ) + if websocket is not None: + try: + await websocket.close() + await websocket.wait_closed() + except Exception as e: + logger.error(e) client.remove_container(container, v=True, force=True) logger.debug(f"Container {container.get('Id')} exited with status code : {str(return_Code['StatusCode'])}") @@ -1542,12 +1581,14 @@ def start(self): self.ingestion_program_exit_code = return_code self.ingestion_program_elapsed_time = elapsed_time logger.info(f"[exited with {logs['returncode']}]") - for key, value in logs.items(): - if key not in ["stdout", "stderr"]: - continue - if value["data"]: - logger.info(f"[{key}]\n{value['data']}") - self._put_file(value["location"], raw_data=value["data"]) + if Settings.SILENT_COMPUTE_WORKER == "false": + for key, value in logs.items(): + if key not in ["stdout", "stderr"]: + continue + if value["data"]: + logger.info(f"[{key}]\n{value['data']}") + self._put_file(value["location"], raw_data=value["data"]) + # set logs of this kind to None, since we handled them already logger.info("Program finished") From bc7db31534d91dbf173b55113aaa76576b4a1341 Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Fri, 3 Jul 2026 16:18:16 +0200 Subject: [PATCH 2/9] rename the No Cleanup env variable, add documentation --- compute_worker/compute_worker.py | 6 +++--- docker-compose.yml | 2 +- .../Compute-Worker-Management---Setup.md | 10 +++++++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index 5d22915dc..d6d48d27a 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -113,7 +113,7 @@ def to_bool(val): COMPETITION_CONTAINER_HTTP_PROXY = get("COMPETITION_CONTAINER_HTTP_PROXY", "") COMPETITION_CONTAINER_HTTPS_PROXY = get("COMPETITION_CONTAINER_HTTPS_PROXY", "") - CODALAB_IGNORE_CLEANUP_STEP = to_bool(get("CODALAB_IGNORE_CLEANUP_STEP")) + COMPUTE_WORKER_NO_CLEANUP = to_bool(get("COMPUTE_WORKER_NO_CLEANUP")) WORKER_BUNDLE_URL_REWRITE = get("WORKER_BUNDLE_URL_REWRITE", "").strip() HUMAN_IN_THE_LOOP = ( @@ -1779,9 +1779,9 @@ def push_output(self): def clean_up(self): self.stop_hitl_http_server() - if Settings.CODALAB_IGNORE_CLEANUP_STEP: + if Settings.COMPUTE_WORKER_NO_CLEANUP: logger.warning( - f"CODALAB_IGNORE_CLEANUP_STEP mode enabled, ignoring clean up of: {self.root_dir}" + f"COMPUTE_WORKER_NO_CLEANUP mode enabled, ignoring clean up of: {self.root_dir}" ) return diff --git a/docker-compose.yml b/docker-compose.yml index 74e9f8818..6e130a557 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -242,7 +242,7 @@ services: environment: - BROKER_URL=pyamqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@${RABBITMQ_HOST}:${RABBITMQ_PORT}// # Make the worker leave behind the submission so we can examine it - - CODALAB_IGNORE_CLEANUP_STEP=1 + - COMPUTE_WORKER_NO_CLEANUP="true" tty: true logging: options: diff --git a/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md b/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md index 8c343ef17..9b677a67f 100644 --- a/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md +++ b/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md @@ -59,11 +59,19 @@ HOST_DIRECTORY=/codabench CONTAINER_ENGINE_EXECUTABLE=docker #USE_GPU=True #GPU_DEVICE=nvidia.com/gpu=all -#HUMAN_IN_THE_LOOP=False# If set to False, the compute worker will never pull for the +#HUMAN_IN_THE_LOOP=False +# If set to False, the compute worker will never pull for the # competition image, the image will need to be downloaded # manually on the host before running submissions. True by default #COMPETITION_ALLOW_IMAGE_PULL=True +# This option removes the ability of the compute worker to send logs to +# codabench, instead writing them locally on disk. Combine with +# COMPUTE_WORKER_NO_CLEANUP=true to stop the worker's cleanup to keep +# all the logs locally only +#SILENT_COMPUTE_WORKER=False +#COMPUTE_WORKER_NO_CLEANUP=False + ####################################################################### # Network # ####################################################################### From cfbe8bf4175fab7d7650d3d8f411c0cd725bba8b Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Mon, 6 Jul 2026 15:53:50 +0200 Subject: [PATCH 3/9] use real boolean values --- compute_worker/compute_worker.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index d6d48d27a..bfdff694c 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -113,7 +113,7 @@ def to_bool(val): COMPETITION_CONTAINER_HTTP_PROXY = get("COMPETITION_CONTAINER_HTTP_PROXY", "") COMPETITION_CONTAINER_HTTPS_PROXY = get("COMPETITION_CONTAINER_HTTPS_PROXY", "") - COMPUTE_WORKER_NO_CLEANUP = to_bool(get("COMPUTE_WORKER_NO_CLEANUP")) + COMPUTE_WORKER_NO_CLEANUP = to_bool(get("COMPUTE_WORKER_NO_CLEANUP", "False")) WORKER_BUNDLE_URL_REWRITE = get("WORKER_BUNDLE_URL_REWRITE", "").strip() HUMAN_IN_THE_LOOP = ( @@ -121,7 +121,7 @@ def to_bool(val): ) COMPETITION_ALLOW_IMAGE_PULL = to_bool(get("COMPETITION_ALLOW_IMAGE_PULL", "True")) - SILENT_COMPUTE_WORKER = get("SILENT_COMPUTE_WORKER", "false").lower() + SILENT_COMPUTE_WORKER = to_bool(get("SILENT_COMPUTE_WORKER", "False")) @@ -376,12 +376,12 @@ def run_wrapper(run_args): except SubmissionException as e: msg = str(e).strip() if msg: - if Settings.SILENT_COMPUTE_WORKER == "true": + if Settings.SILENT_COMPUTE_WORKER: msg = f"Submission failed: {msg}. Contact the Organizer for more details." else: msg = f"Submission failed: {msg}. See logs for more details." else: - if Settings.SILENT_COMPUTE_WORKER == "true": + if Settings.SILENT_COMPUTE_WORKER: msg = "Submission failed. Contact the Organizer for more details." else: msg = "Submission failed. See logs for more details." @@ -610,7 +610,7 @@ async def watch_detailed_results(self): def push_logs(self): """Upload any collected logs, even in case of crash. """ - if Settings.SILENT_COMPUTE_WORKER == "false": + if Settings.SILENT_COMPUTE_WORKER == False: try: for kind, logs in (self.logs or {}).items(): for stream_key in ("stdout", "stderr"): @@ -795,7 +795,7 @@ def _get_container_image(self, image_name): self._update_submission(docker_pull_fail_data) # Send error through web socket to the frontend asyncio.run(self._send_data_through_socket(str(pull_error))) - if Settings.SILENT_COMPUTE_WORKER == "true": + if Settings.SILENT_COMPUTE_WORKER: raise DockerImagePullException( f"Pull for {image_name} failed! Contact the Organizer for more details." ) @@ -1037,7 +1037,7 @@ async def _run_container_engine_cmd(self, container, kind): websocket = None # Do not create a websocket if the real time logs are not wanted (Silent Compute Worker) - if Settings.SILENT_COMPUTE_WORKER == "false": + if Settings.SILENT_COMPUTE_WORKER == False: try: websocket_url = f"{self.websocket_url}?kind={kind}" logger.debug(f"Connecting to {websocket_url} for container {str(container.get('Id'))}") @@ -1078,7 +1078,7 @@ async def _run_container_engine_cmd(self, container, kind): if log[0] is not None: stdout_chunks.append(log[0]) logger.info(log[0].decode()) - if Settings.SILENT_COMPUTE_WORKER == "false": + if Settings.SILENT_COMPUTE_WORKER == False: try: if websocket is not None: await websocket.send( @@ -1091,7 +1091,7 @@ async def _run_container_engine_cmd(self, container, kind): elif log[1] is not None: stderr_chunks.append(log[1]) logger.error(log[1].decode()) - if Settings.SILENT_COMPUTE_WORKER == "false": + if Settings.SILENT_COMPUTE_WORKER == False: try: if websocket is not None: await websocket.send( @@ -1115,7 +1115,7 @@ async def _run_container_engine_cmd(self, container, kind): return_Code = client.wait(container) logs_Unified = (b"".join(stdout_chunks), b"".join(stderr_chunks)) - if Settings.SILENT_COMPUTE_WORKER == "false": + if Settings.SILENT_COMPUTE_WORKER == False: logger.debug( f"WORKER_MARKER: Disconnecting from {websocket_url}, program counter = {self.completed_program_counter}" ) @@ -1484,6 +1484,7 @@ def start(self): self._run_program_directory(kind=ProgramKind.INGESTION_PROGRAM, program_dir=ingestion_program_dir), ]) + logger.info(tasks) gathered_tasks = asyncio.gather(*tasks, return_exceptions=True) task_results = [] # will store results/exceptions from gather @@ -1581,7 +1582,7 @@ def start(self): self.ingestion_program_exit_code = return_code self.ingestion_program_elapsed_time = elapsed_time logger.info(f"[exited with {logs['returncode']}]") - if Settings.SILENT_COMPUTE_WORKER == "false": + if Settings.SILENT_COMPUTE_WORKER == False: for key, value in logs.items(): if key not in ["stdout", "stderr"]: continue From a173617546c7d12e71ea9b3031bea0bc69b23bf0 Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Mon, 6 Jul 2026 16:50:20 +0200 Subject: [PATCH 4/9] update logs_loguru to inclue new tasks variable names to color them --- src/settings/logs_loguru.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/settings/logs_loguru.py b/src/settings/logs_loguru.py index 28b2cf075..137ff83c9 100644 --- a/src/settings/logs_loguru.py +++ b/src/settings/logs_loguru.py @@ -130,7 +130,17 @@ def colorize_run_args(json_str): json_str, ) json_str = re.sub( - r'("ingestion_program": ")(.*?)(",)', + r'("ingestion_program_data": ")(.*?)(",)', + rf"\1{yellow}\2{reset}\3{lineskip}", + json_str, + ) + json_str = re.sub( + r'("submission_data": ")(.*?)(",)', + rf"\1{yellow}\2{reset}\3{lineskip}", + json_str, + ) + json_str = re.sub( + r'("scoring_program_data": ")(.*?)(",)', rf"\1{yellow}\2{reset}\3{lineskip}", json_str, ) From bd1d1d7c72532a317d0a01bc4659f048fa241ceb Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Mon, 6 Jul 2026 16:51:04 +0200 Subject: [PATCH 5/9] change variable name to use boolean --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6e130a557..4b2949dbb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -242,7 +242,7 @@ services: environment: - BROKER_URL=pyamqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@${RABBITMQ_HOST}:${RABBITMQ_PORT}// # Make the worker leave behind the submission so we can examine it - - COMPUTE_WORKER_NO_CLEANUP="true" + - COMPUTE_WORKER_NO_CLEANUP=True tty: true logging: options: From 5e2d17a5306a8285448aedb6424ef8369548632f Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Tue, 7 Jul 2026 14:44:15 +0200 Subject: [PATCH 6/9] fix some syntax --- compute_worker/compute_worker.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index bfdff694c..e6e255e13 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -377,12 +377,12 @@ def run_wrapper(run_args): msg = str(e).strip() if msg: if Settings.SILENT_COMPUTE_WORKER: - msg = f"Submission failed: {msg}. Contact the Organizer for more details." + msg = f"Submission failed: {msg}. Contact the Organizer(s) for more details." else: msg = f"Submission failed: {msg}. See logs for more details." else: if Settings.SILENT_COMPUTE_WORKER: - msg = "Submission failed. Contact the Organizer for more details." + msg = "Submission failed. Contact the Organizer(s) for more details." else: msg = "Submission failed. See logs for more details." run._update_status(SubmissionStatus.FAILED, extra_information=msg) @@ -797,7 +797,7 @@ def _get_container_image(self, image_name): asyncio.run(self._send_data_through_socket(str(pull_error))) if Settings.SILENT_COMPUTE_WORKER: raise DockerImagePullException( - f"Pull for {image_name} failed! Contact the Organizer for more details." + f"Pull for {image_name} failed! Contact the Organizer(s) for more details." ) else: raise DockerImagePullException( @@ -957,7 +957,8 @@ def _create_container( "SYS_CHROOT", ] - # Configure whether or not we use the GPU. Also setting auto_remove to False because + # Configure whether or not we use the GPU. Also setting auto_remove to False because removing too fast + # can bug out the worker (can't get the logs fast enough) if Settings.CONTAINER_ENGINE_EXECUTABLE == Settings.DOCKER: security_options = ["no-new-privileges"] else: From 1e4d018770aaa257051375f669c76a55dfe62f17 Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Thu, 27 Aug 2026 15:51:02 +0200 Subject: [PATCH 7/9] add option to forbid the compute worker from sending predictions to the codabench instance storage --- compute_worker/compute_worker.py | 55 ++++++++++++++----- .../Compute-Worker-Management---Setup.md | 7 ++- ...Compute-worker-installation-with-Podman.md | 12 ++++ 3 files changed, 58 insertions(+), 16 deletions(-) diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index e6e255e13..afd9dde02 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -121,7 +121,8 @@ def to_bool(val): ) COMPETITION_ALLOW_IMAGE_PULL = to_bool(get("COMPETITION_ALLOW_IMAGE_PULL", "True")) - SILENT_COMPUTE_WORKER = to_bool(get("SILENT_COMPUTE_WORKER", "False")) + COMPUTE_WORKER_DISABLE_LOG_UPLOAD = to_bool(get("COMPUTE_WORKER_DISABLE_LOG_UPLOAD", "False")) + COMPUTE_WORKER_DISABLE_PREDICTION_UPLOAD = to_bool(get("COMPUTE_WORKER_DISABLE_PREDICTION_UPLOAD", "False")) @@ -175,6 +176,10 @@ class SubmissionStatus: f"{'with GPU capabilities: ' + Settings.GPU_DEVICE if Settings.USE_GPU else 'without GPU capabilities'}. " f"Network disabled for the competition container is set to {Settings.COMPETITION_CONTAINER_NETWORK_DISABLED}" ) +if Settings.COMPUTE_WORKER_DISABLE_PREDICTION_UPLOAD: + logger.warning("COMPUTE_WORKER_DISABLE_PREDICTION_UPLOAD is set to True, setting COMPUTE_WORKER_NO_CLEANUP to True") + Settings.COMPUTE_WORKER_NO_CLEANUP = True + # Intializing client # NOTE: CONTAINER_SOCKET is set in Settings based on CONTAINER_ENGINE_EXECUTABLE which must has either podman or docker @@ -376,12 +381,12 @@ def run_wrapper(run_args): except SubmissionException as e: msg = str(e).strip() if msg: - if Settings.SILENT_COMPUTE_WORKER: + if Settings.COMPUTE_WORKER_DISABLE_LOG_UPLOAD: msg = f"Submission failed: {msg}. Contact the Organizer(s) for more details." else: msg = f"Submission failed: {msg}. See logs for more details." else: - if Settings.SILENT_COMPUTE_WORKER: + if Settings.COMPUTE_WORKER_DISABLE_LOG_UPLOAD: msg = "Submission failed. Contact the Organizer(s) for more details." else: msg = "Submission failed. See logs for more details." @@ -502,10 +507,19 @@ def __init__(self, run_args): self.run_related_name = ( f"uPK-{run_args['user_pk']}_sID-{run_args['id']}" ) + if run_args["is_scoring"]: + task_type = "scoring_program" + else: + task_type = "ingestion" + # Directories for the run self.watch = True self.completed_program_counter = 0 - self.root_dir = tempfile.mkdtemp(prefix=f'{self.run_related_name}__', dir=Settings.BASE_DIR) + # Create the folder then save the path to root_dir + self.submission_run_directory = Settings.BASE_DIR + f'{self.run_related_name}__' + os.mkdir(self.submission_run_directory + task_type, mode=0o700) + self.root_dir = self.submission_run_directory + task_type + self.bundle_dir = os.path.join(self.root_dir, "bundles") self.input_dir = os.path.join(self.root_dir, "input") self.output_dir = os.path.join(self.root_dir, "output") @@ -519,7 +533,10 @@ def __init__(self, run_args): self.submissions_api_url = run_args["submissions_api_url"] self.container_image = run_args["docker_image"] self.secret = run_args["secret"] - self.prediction_result = run_args["prediction_result"] + if Settings.COMPUTE_WORKER_DISABLE_PREDICTION_UPLOAD: + self.prediction_result = "Prediction Upload Disabled." + else: + self.prediction_result = run_args["prediction_result"] self.scoring_result = run_args.get("scoring_result") self.execution_time_limit = run_args["execution_time_limit"] # ----- HITL ------ @@ -610,7 +627,7 @@ async def watch_detailed_results(self): def push_logs(self): """Upload any collected logs, even in case of crash. """ - if Settings.SILENT_COMPUTE_WORKER == False: + if not Settings.COMPUTE_WORKER_DISABLE_LOG_UPLOAD: try: for kind, logs in (self.logs or {}).items(): for stream_key in ("stdout", "stderr"): @@ -795,7 +812,7 @@ def _get_container_image(self, image_name): self._update_submission(docker_pull_fail_data) # Send error through web socket to the frontend asyncio.run(self._send_data_through_socket(str(pull_error))) - if Settings.SILENT_COMPUTE_WORKER: + if Settings.COMPUTE_WORKER_DISABLE_LOG_UPLOAD: raise DockerImagePullException( f"Pull for {image_name} failed! Contact the Organizer(s) for more details." ) @@ -1038,7 +1055,7 @@ async def _run_container_engine_cmd(self, container, kind): websocket = None # Do not create a websocket if the real time logs are not wanted (Silent Compute Worker) - if Settings.SILENT_COMPUTE_WORKER == False: + if not Settings.COMPUTE_WORKER_DISABLE_LOG_UPLOAD: try: websocket_url = f"{self.websocket_url}?kind={kind}" logger.debug(f"Connecting to {websocket_url} for container {str(container.get('Id'))}") @@ -1079,7 +1096,7 @@ async def _run_container_engine_cmd(self, container, kind): if log[0] is not None: stdout_chunks.append(log[0]) logger.info(log[0].decode()) - if Settings.SILENT_COMPUTE_WORKER == False: + if not Settings.COMPUTE_WORKER_DISABLE_LOG_UPLOAD: try: if websocket is not None: await websocket.send( @@ -1092,7 +1109,7 @@ async def _run_container_engine_cmd(self, container, kind): elif log[1] is not None: stderr_chunks.append(log[1]) logger.error(log[1].decode()) - if Settings.SILENT_COMPUTE_WORKER == False: + if not Settings.COMPUTE_WORKER_DISABLE_LOG_UPLOAD: try: if websocket is not None: await websocket.send( @@ -1116,7 +1133,7 @@ async def _run_container_engine_cmd(self, container, kind): return_Code = client.wait(container) logs_Unified = (b"".join(stdout_chunks), b"".join(stderr_chunks)) - if Settings.SILENT_COMPUTE_WORKER == False: + if not Settings.COMPUTE_WORKER_DISABLE_LOG_UPLOAD: logger.debug( f"WORKER_MARKER: Disconnecting from {websocket_url}, program counter = {self.completed_program_counter}" ) @@ -1407,15 +1424,22 @@ def prepare(self): (self.input_data, "input_data"), (self.reference_data, "input/ref"), ] - if self.is_scoring: + if self.is_scoring and not Settings.COMPUTE_WORKER_DISABLE_PREDICTION_UPLOAD: # Send along submission result so scoring_program can get access bundles += [(self.prediction_result, "input/res")] + elif self.is_scoring: + bundles += [("local_prediction_results", "submission")] for url, path in bundles: if url is not None: # At the moment let's just cache input & reference data cache_this_bundle = path in ("input_data", "input/ref") - zip_file = self._get_bundle(url, path, cache=cache_this_bundle) + if url == "local_prediction_results" and self.is_scoring: + submission_run_directory_ingestion = self.submission_run_directory + "ingestion/output/" + submission_run_directory_scoring = self.submission_run_directory + "scoring_program/submission/" + shutil.copytree(submission_run_directory_ingestion, submission_run_directory_scoring, dirs_exist_ok=True) + else: + zip_file = self._get_bundle(url, path, cache=cache_this_bundle) # Computing checksum of the submission file during ingestion run if url == self.submission_data and not self.is_scoring: @@ -1583,7 +1607,7 @@ def start(self): self.ingestion_program_exit_code = return_code self.ingestion_program_elapsed_time = elapsed_time logger.info(f"[exited with {logs['returncode']}]") - if Settings.SILENT_COMPUTE_WORKER == False: + if Settings.COMPUTE_WORKER_DISABLE_LOG_UPLOAD == False: for key, value in logs.items(): if key not in ["stdout", "stderr"]: continue @@ -1775,7 +1799,8 @@ def push_output(self): raise SubmissionException("Failed to write metadata file.") if not self.is_scoring: - self._put_dir(self.prediction_result, self.output_dir) + if not Settings.COMPUTE_WORKER_DISABLE_PREDICTION_UPLOAD: + self._put_dir(self.prediction_result, self.output_dir) else: self._put_dir(self.scoring_result, self.output_dir) diff --git a/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md b/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md index 9b677a67f..778554172 100644 --- a/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md +++ b/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md @@ -69,7 +69,12 @@ CONTAINER_ENGINE_EXECUTABLE=docker # codabench, instead writing them locally on disk. Combine with # COMPUTE_WORKER_NO_CLEANUP=true to stop the worker's cleanup to keep # all the logs locally only -#SILENT_COMPUTE_WORKER=False +#COMPUTE_WORKER_DISABLE_LOG_UPLOAD=False +# Stop the predictions from being sent to Codabench. +# This option requires only having one compute worker for ingestion +# and scoring. +#COMPUTE_WORKER_DISABLE_PREDICTION_UPLOAD=False + #COMPUTE_WORKER_NO_CLEANUP=False ####################################################################### diff --git a/documentation/docs/Organizers/Running_a_benchmark/Compute-worker-installation-with-Podman.md b/documentation/docs/Organizers/Running_a_benchmark/Compute-worker-installation-with-Podman.md index 6e2729157..153aa063c 100644 --- a/documentation/docs/Organizers/Running_a_benchmark/Compute-worker-installation-with-Podman.md +++ b/documentation/docs/Organizers/Running_a_benchmark/Compute-worker-installation-with-Podman.md @@ -37,6 +37,18 @@ HOST_DIRECTORY=/codabench CONTAINER_ENGINE_EXECUTABLE=podman #USE_GPU=True #GPU_DEVICE=nvidia.com/gpu=all +#HUMAN_IN_THE_LOOP=False +# This option removes the ability of the compute worker to send logs to +# codabench, instead writing them locally on disk. Combine with +# COMPUTE_WORKER_NO_CLEANUP=true to stop the worker's cleanup to keep +# all the logs locally only +#COMPUTE_WORKER_DISABLE_LOG_UPLOAD=False +# Stop the predictions from being sent to Codabench. +# This option requires only having one compute worker for ingestion +# and scoring. +#COMPUTE_WORKER_DISABLE_PREDICTION_UPLOAD=False + +#COMPUTE_WORKER_NO_CLEANUP=False ####################################################################### # Network # From 974b95d51aec19350edecbf92e1a42aa19fd3fd1 Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Tue, 8 Sep 2026 14:38:10 +0200 Subject: [PATCH 8/9] add better coloration for some logs in the compute worker --- src/settings/logs_loguru.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/settings/logs_loguru.py b/src/settings/logs_loguru.py index 137ff83c9..99069df1d 100644 --- a/src/settings/logs_loguru.py +++ b/src/settings/logs_loguru.py @@ -75,6 +75,11 @@ def colorize_run_args(json_str): rf"\1{green}\2{reset}\3{lineskip}", json_str, ) + json_str = re.sub( + r'("human_in_the_loop": )(.*?)(,)', + rf"\1{green}\2{reset}\3{lineskip}", + json_str, + ) json_str = re.sub( r'("is_scoring": )(.*?)(,)', rf"\1{green}\2{reset}\3{lineskip}", json_str ) From 5021e74a9fd4fb70bc71aab3b8fb9949123b55a0 Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Tue, 8 Sep 2026 15:11:05 +0200 Subject: [PATCH 9/9] update documentation --- .../Compute-Worker-Management---Setup.md | 2 ++ .../Compute-worker-installation-with-Podman.md | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md b/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md index 778554172..660302973 100644 --- a/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md +++ b/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md @@ -60,6 +60,7 @@ CONTAINER_ENGINE_EXECUTABLE=docker #USE_GPU=True #GPU_DEVICE=nvidia.com/gpu=all #HUMAN_IN_THE_LOOP=False + # If set to False, the compute worker will never pull for the # competition image, the image will need to be downloaded # manually on the host before running submissions. True by default @@ -70,6 +71,7 @@ CONTAINER_ENGINE_EXECUTABLE=docker # COMPUTE_WORKER_NO_CLEANUP=true to stop the worker's cleanup to keep # all the logs locally only #COMPUTE_WORKER_DISABLE_LOG_UPLOAD=False + # Stop the predictions from being sent to Codabench. # This option requires only having one compute worker for ingestion # and scoring. diff --git a/documentation/docs/Organizers/Running_a_benchmark/Compute-worker-installation-with-Podman.md b/documentation/docs/Organizers/Running_a_benchmark/Compute-worker-installation-with-Podman.md index 153aa063c..b357684a8 100644 --- a/documentation/docs/Organizers/Running_a_benchmark/Compute-worker-installation-with-Podman.md +++ b/documentation/docs/Organizers/Running_a_benchmark/Compute-worker-installation-with-Podman.md @@ -38,11 +38,18 @@ CONTAINER_ENGINE_EXECUTABLE=podman #USE_GPU=True #GPU_DEVICE=nvidia.com/gpu=all #HUMAN_IN_THE_LOOP=False + +# If set to False, the compute worker will never pull for the +# competition image, the image will need to be downloaded +# manually on the host before running submissions. True by default +#COMPETITION_ALLOW_IMAGE_PULL=True + # This option removes the ability of the compute worker to send logs to # codabench, instead writing them locally on disk. Combine with # COMPUTE_WORKER_NO_CLEANUP=true to stop the worker's cleanup to keep # all the logs locally only #COMPUTE_WORKER_DISABLE_LOG_UPLOAD=False + # Stop the predictions from being sent to Codabench. # This option requires only having one compute worker for ingestion # and scoring.