[fine_tuning] Enable fsdp cpu offloading

projects/cluster/toolbox/cluster_download_to_pvc/templates/pod.yml.j2

@@ -36,6 +36,8 @@ spec:
       subPath: entrypoint.sh
     - name: storage
       mountPath: /storage
+  serviceAccount: rhoai-tuning-test-sa
+  serviceAccountName: rhoai-tuning-test-sa
   volumes:
 {% if cluster_download_to_pvc_creds | length %}
   - name: download-secret

projects/fine_tuning/testing/config.yaml

@@ -107,18 +107,13 @@ ci_presets:
     - name: granite-family
       model_name:
       - ibm-granite/granite-3b-code-instruct
-      - instructlab/granite-7b-lab
-      - ibm-granite/granite-8b-code-base
-      - ibm-granite/granite-8b-code-instruct
       hyper_parameters:
         per_device_train_batch_size: 1
     - name: llama-family
       model_name:
-      - meta-llama/Meta-Llama-3-8B-Instruct
       - meta-llama/llama-2-7b-hf
-      - meta-llama/Llama-2-13b-chat-hf
       hyper_parameters:
-        per_device_train_batch_size: [1, 2]
+        per_device_train_batch_size: 1
 
   ibm_80gb_models:
     extends: [hf_evaluation, dgx_small_footprint]

projects/fine_tuning/testing/prepare_finetuning.py

@@ -40,6 +40,7 @@ def prepare_gpu():
 
 
 def prepare_rhoai():
+    return
     if not PSAP_ODS_SECRET_PATH.exists():
         raise RuntimeError(f"Path with the secrets (PSAP_ODS_SECRET_PATH={PSAP_ODS_SECRET_PATH}) does not exists.")
 
@@ -68,6 +69,7 @@ def prepare_rhoai():
 
 
 def set_namespace_annotations():
+    return
     metal = config.ci_artifacts.get_config("clusters.sutest.is_metal")
     dedicated = config.ci_artifacts.get_config("clusters.sutest.compute.dedicated")
     namespace = config.ci_artifacts.get_config("tests.fine_tuning.namespace")

projects/fine_tuning/toolbox/fine_tuning_run_fine_tuning_job/templates/base_config.yaml.j2

@@ -25,6 +25,7 @@ accelerate_launch_args:
   tpu_use_sudo: false
 {% if (fine_tuning_run_fine_tuning_job_gpu or 1) > 1 %}
   use_fsdp: true
+  # fsdp_offload_params: true
 {% endif %}
 # below are the *default* values. They might be overwritten when 'config_final.json' is generated.
 

projects/fine_tuning/toolbox/fine_tuning_run_fine_tuning_job/templates/fine_tuning_job.yaml.j2

@@ -54,7 +54,7 @@ spec:
               mountPath: /mnt/entrypoint
             - name: config-volume
               mountPath: /mnt/config
-            - name: output-volume
+            - name: scratch-volume
               mountPath: /mnt/output
             resources:
               requests: &request_block
@@ -79,8 +79,18 @@ spec:
           - name: entrypoint-volume
             configMap:
               name: {{ job_name_safe }}-entrypoint
-          - name: output-volume
-            emptyDir: {}
+          # - name: output-volume
+          #   emptyDir: {}
+          - name: scratch-volume
+            ephemeral:
+              volumeClaimTemplate:
+                metadata:
+                spec:
+                  accessModes: [ "ReadWriteOnce" ]
+                  resources:
+                    requests:
+                      storage: 500Gi
+
 {% if fine_tuning_run_fine_tuning_job_worker_replicas > 0 %}
     Worker:
       replicas: {{ fine_tuning_run_fine_tuning_job_worker_replicas }}

projects/local_ci/testing/config.yaml

@@ -1,13 +1,13 @@
 base_image:
-  namespace: topsail
+  namespace: rhoai-tuning-test
   imagestream: topsail
   repo:
     url: https://github.com/openshift-psap/topsail/
     ref: main
     dockerfile_path: build/Dockerfile
   user:
-    service_account: topsail
-    role: cluster-admin
+    service_account: rhoai-tuning-test-sa
+    role: rhoai-tuning-test-admin-role
 secrets:
   psap-ods-secret:
     env_key: PSAP_ODS_SECRET_PATH

projects/local_ci/testing/run

@@ -73,8 +73,8 @@ def local_ci_prepare():
     # Prepare the ServiceAccount
     #
 
-    run.run(f"oc create serviceaccount {service_account} -n {namespace} --dry-run=client -oyaml | oc apply -f-")
-    run.run(f"oc adm policy add-cluster-role-to-user {role} -z {service_account} -n {namespace}")
+    # run.run(f"oc create serviceaccount {service_account} -n {namespace} --dry-run=client -oyaml | oc apply -f-")
+    # run.run(f"oc adm policy add-cluster-role-to-user {role} -z {service_account} -n {namespace}")
 
     #
     # Prepare the Secrets