runtime_config.yaml

# its contents configure the TGIS server & caikit
jvm_options: []

runtime:
  library: caikit_nlp
  lazy_load_local_models: true
  batching:
    standalone-model:
      size: 0 # Set to batch size for batching

model_management:
  finders:
    default:
      type: LOCAL
    remote_tgis:
      type: TGIS-AUTO
      config:
        test_connection: true
  initializers:
    default:
      type: LOCAL
      config:
        backend_priority:
          - type: TGIS
            config:
              local:
                load_timeout: 120
                grpc_port: null
                http_port: null
                health_poll_delay: 1.0
              remote_models:
                flan-t5-xl:
                  hostname: localhost:8033
                  prompt_dir: tgis_prompts
                llama-70b:
                  hostname: localhost:8034
                  prompt_dir: tgis_prompts

              connection:
                hostname: "foo.{model_id}:1234"
                ca_cert_file: null
                client_cert_file: null
                client_key_file: null