diff --git a/batch/snippets/pom.xml b/batch/snippets/pom.xml new file mode 100644 index 00000000000..dafd04a2bd9 --- /dev/null +++ b/batch/snippets/pom.xml @@ -0,0 +1,49 @@ + + + 4.0.0 + com.google.cloud + -snippets + jar + Google Google Cloud Batch Snippets + https://github.com/googleapis/java-batch + + + + com.google.cloud.samples + shared-configuration + 1.2.0 + + + + 1.8 + 1.8 + UTF-8 + + + + + + com.google.cloud + google-cloud-batch + 0.3.2 + + + + junit + junit + 4.13.2 + test + + + com.google.truth + truth + 1.1.3 + test + + + diff --git a/batch/snippets/src/main/java/CreateWithContainerNoMounting.java b/batch/snippets/src/main/java/CreateWithContainerNoMounting.java new file mode 100644 index 00000000000..0947544c47d --- /dev/null +++ b/batch/snippets/src/main/java/CreateWithContainerNoMounting.java @@ -0,0 +1,140 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// [START batch_create_container_job] + +import com.google.cloud.batch.v1.AllocationPolicy; +import com.google.cloud.batch.v1.AllocationPolicy.InstancePolicy; +import com.google.cloud.batch.v1.AllocationPolicy.InstancePolicyOrTemplate; +import com.google.cloud.batch.v1.BatchServiceClient; +import com.google.cloud.batch.v1.ComputeResource; +import com.google.cloud.batch.v1.CreateJobRequest; +import com.google.cloud.batch.v1.Job; +import com.google.cloud.batch.v1.LogsPolicy; +import com.google.cloud.batch.v1.LogsPolicy.Destination; +import com.google.cloud.batch.v1.Runnable; +import com.google.cloud.batch.v1.Runnable.Container; +import com.google.cloud.batch.v1.TaskGroup; +import com.google.cloud.batch.v1.TaskSpec; +import com.google.protobuf.Duration; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public class CreateWithContainerNoMounting { + + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException, TimeoutException { + // TODO(developer): Replace these variables before running the sample. + // Project ID or project number of the Cloud project you want to use. + String projectId = "YOUR_PROJECT_ID"; + + // Name of the region you want to use to run the job. Regions that are + // available for Batch are listed on: https://cloud.google.com/batch/docs/get-started#locations + String region = "europe-central2"; + + // The name of the job that will be created. + // It needs to be unique for each project and region pair. + String jobName = "JOB_NAME"; + + createContainerJob(projectId, region, jobName); + } + + // This method shows how to create a sample Batch Job that will run a simple command inside a + // container on Cloud Compute instances. + public static void createContainerJob(String projectId, String region, String jobName) + throws IOException, ExecutionException, InterruptedException, TimeoutException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the `batchServiceClient.close()` method on the client to safely + // clean up any remaining background resources. + try (BatchServiceClient batchServiceClient = BatchServiceClient.create()) { + + // Define what will be done as part of the job. + Runnable runnable = + Runnable.newBuilder() + .setContainer( + Container.newBuilder() + .setImageUri("gcr.io/google-containers/busybox") + .setEntrypoint("/bin/sh") + .addCommands("-c") + .addCommands( + "echo Hello world! This is task ${BATCH_TASK_INDEX}. " + + "This job has a total of ${BATCH_TASK_COUNT} tasks.") + .build()) + .build(); + + // We can specify what resources are requested by each task. + ComputeResource computeResource = + ComputeResource.newBuilder() + // In milliseconds per cpu-second. This means the task requires 2 whole CPUs. + .setCpuMilli(2000) + // In MiB. + .setMemoryMib(16) + .build(); + + TaskSpec task = + TaskSpec.newBuilder() + // Jobs can be divided into tasks. In this case, we have only one task. + .addRunnables(runnable) + .setComputeResource(computeResource) + .setMaxRetryCount(2) + .setMaxRunDuration(Duration.newBuilder().setSeconds(3600).build()) + .build(); + + // Tasks are grouped inside a job using TaskGroups. + TaskGroup taskGroup = TaskGroup.newBuilder().setTaskCount(4).setTaskSpec(task).build(); + + // Policies are used to define on what kind of virtual machines the tasks will run on. + // In this case, we tell the system to use "e2-standard-4" machine type. + // Read more about machine types here: https://cloud.google.com/compute/docs/machine-types + InstancePolicy instancePolicy = + InstancePolicy.newBuilder().setMachineType("e2-standard-4").build(); + + AllocationPolicy allocationPolicy = + AllocationPolicy.newBuilder() + .addInstances(InstancePolicyOrTemplate.newBuilder().setPolicy(instancePolicy).build()) + .build(); + + Job job = + Job.newBuilder() + .addTaskGroups(taskGroup) + .setAllocationPolicy(allocationPolicy) + .putLabels("env", "testing") + .putLabels("type", "container") + // We use Cloud Logging as it's an out of the box available option. + .setLogsPolicy( + LogsPolicy.newBuilder().setDestination(Destination.CLOUD_LOGGING).build()) + .build(); + + CreateJobRequest createJobRequest = + CreateJobRequest.newBuilder() + // The job's parent is the region in which the job will run. + .setParent(String.format("projects/%s/locations/%s", projectId, region)) + .setJob(job) + .setJobId(jobName) + .build(); + + Job result = + batchServiceClient + .createJobCallable() + .futureCall(createJobRequest) + .get(5, TimeUnit.MINUTES); + + System.out.printf("Successfully created the job: %s", result.getName()); + } + } +} +// [END batch_create_container_job] diff --git a/batch/snippets/src/main/java/CreateWithScriptNoMounting.java b/batch/snippets/src/main/java/CreateWithScriptNoMounting.java new file mode 100644 index 00000000000..70c86813302 --- /dev/null +++ b/batch/snippets/src/main/java/CreateWithScriptNoMounting.java @@ -0,0 +1,141 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// [START batch_create_script_job] + +import com.google.cloud.batch.v1.AllocationPolicy; +import com.google.cloud.batch.v1.AllocationPolicy.InstancePolicy; +import com.google.cloud.batch.v1.AllocationPolicy.InstancePolicyOrTemplate; +import com.google.cloud.batch.v1.BatchServiceClient; +import com.google.cloud.batch.v1.ComputeResource; +import com.google.cloud.batch.v1.CreateJobRequest; +import com.google.cloud.batch.v1.Job; +import com.google.cloud.batch.v1.LogsPolicy; +import com.google.cloud.batch.v1.LogsPolicy.Destination; +import com.google.cloud.batch.v1.Runnable; +import com.google.cloud.batch.v1.Runnable.Script; +import com.google.cloud.batch.v1.TaskGroup; +import com.google.cloud.batch.v1.TaskSpec; +import com.google.protobuf.Duration; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public class CreateWithScriptNoMounting { + + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException, TimeoutException { + // TODO(developer): Replace these variables before running the sample. + // Project ID or project number of the Cloud project you want to use. + String projectId = "YOUR_PROJECT_ID"; + + // Name of the region you want to use to run the job. Regions that are + // available for Batch are listed on: https://cloud.google.com/batch/docs/get-started#locations + String region = "europe-central2"; + + // The name of the job that will be created. + // It needs to be unique for each project and region pair. + String jobName = "JOB_NAME"; + + createScriptJob(projectId, region, jobName); + } + + // This method shows how to create a sample Batch Job that will run + // a simple command on Cloud Compute instances. + public static void createScriptJob(String projectId, String region, String jobName) + throws IOException, ExecutionException, InterruptedException, TimeoutException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the `batchServiceClient.close()` method on the client to safely + // clean up any remaining background resources. + try (BatchServiceClient batchServiceClient = BatchServiceClient.create()) { + + // Define what will be done as part of the job. + Runnable runnable = + Runnable.newBuilder() + .setScript( + Script.newBuilder() + .setText( + "echo Hello world! This is task ${BATCH_TASK_INDEX}. " + + "This job has a total of ${BATCH_TASK_COUNT} tasks.") + // You can also run a script from a file. Just remember, that needs to be a + // script that's already on the VM that will be running the job. + // Using setText() and setPath() is mutually exclusive. + // .setPath("/tmp/test.sh") + .build()) + .build(); + + // We can specify what resources are requested by each task. + ComputeResource computeResource = + ComputeResource.newBuilder() + // In milliseconds per cpu-second. This means the task requires 2 whole CPUs. + .setCpuMilli(2000) + // In MiB. + .setMemoryMib(16) + .build(); + + TaskSpec task = + TaskSpec.newBuilder() + // Jobs can be divided into tasks. In this case, we have only one task. + .addRunnables(runnable) + .setComputeResource(computeResource) + .setMaxRetryCount(2) + .setMaxRunDuration(Duration.newBuilder().setSeconds(3600).build()) + .build(); + + // Tasks are grouped inside a job using TaskGroups. + TaskGroup taskGroup = TaskGroup.newBuilder().setTaskCount(4).setTaskSpec(task).build(); + + // Policies are used to define on what kind of virtual machines the tasks will run on. + // In this case, we tell the system to use "e2-standard-4" machine type. + // Read more about machine types here: https://cloud.google.com/compute/docs/machine-types + InstancePolicy instancePolicy = + InstancePolicy.newBuilder().setMachineType("e2-standard-4").build(); + + AllocationPolicy allocationPolicy = + AllocationPolicy.newBuilder() + .addInstances(InstancePolicyOrTemplate.newBuilder().setPolicy(instancePolicy).build()) + .build(); + + Job job = + Job.newBuilder() + .addTaskGroups(taskGroup) + .setAllocationPolicy(allocationPolicy) + .putLabels("env", "testing") + .putLabels("type", "script") + // We use Cloud Logging as it's an out of the box available option. + .setLogsPolicy( + LogsPolicy.newBuilder().setDestination(Destination.CLOUD_LOGGING).build()) + .build(); + + CreateJobRequest createJobRequest = + CreateJobRequest.newBuilder() + // The job's parent is the region in which the job will run. + .setParent(String.format("projects/%s/locations/%s", projectId, region)) + .setJob(job) + .setJobId(jobName) + .build(); + + Job result = + batchServiceClient + .createJobCallable() + .futureCall(createJobRequest) + .get(5, TimeUnit.MINUTES); + + System.out.printf("Successfully created the job: %s", result.getName()); + } + } +} +// [END batch_create_script_job] diff --git a/batch/snippets/src/main/java/DeleteJob.java b/batch/snippets/src/main/java/DeleteJob.java new file mode 100644 index 00000000000..d9f1a809668 --- /dev/null +++ b/batch/snippets/src/main/java/DeleteJob.java @@ -0,0 +1,57 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// [START batch_delete_job] + +import com.google.cloud.batch.v1.BatchServiceClient; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public class DeleteJob { + + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException, TimeoutException { + // TODO(developer): Replace these variables before running the sample. + // Project ID or project number of the Cloud project you want to use. + String projectId = "YOUR_PROJECT_ID"; + + // Name of the region hosts the job. + String region = "europe-central2"; + + // The name of the job that you want to delete. + String jobName = "JOB_NAME"; + + deleteJob(projectId, region, jobName); + } + + // Triggers the deletion of a Job. + public static void deleteJob(String projectId, String region, String jobName) + throws IOException, ExecutionException, InterruptedException, TimeoutException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the `batchServiceClient.close()` method on the client to safely + // clean up any remaining background resources. + try (BatchServiceClient batchServiceClient = BatchServiceClient.create()) { + + // Construct the parent path of the job. + String name = String.format("projects/%s/locations/%s/jobs/%s", projectId, region, jobName); + + batchServiceClient.deleteJobAsync(name).get(5, TimeUnit.MINUTES); + System.out.printf("Delete the job: %s", jobName); + } + } +} +// [END batch_delete_job] diff --git a/batch/snippets/src/main/java/GetJob.java b/batch/snippets/src/main/java/GetJob.java new file mode 100644 index 00000000000..c7637f70bb9 --- /dev/null +++ b/batch/snippets/src/main/java/GetJob.java @@ -0,0 +1,58 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// [START batch_get_job] + +import com.google.cloud.batch.v1.BatchServiceClient; +import com.google.cloud.batch.v1.Job; +import com.google.cloud.batch.v1.JobName; +import java.io.IOException; + +public class GetJob { + + public static void main(String[] args) throws IOException { + // TODO(developer): Replace these variables before running the sample. + // Project ID or project number of the Cloud project you want to use. + String projectId = "YOUR_PROJECT_ID"; + + // Name of the region hosts the job. + String region = "europe-central2"; + + // The name of the job you want to retrieve information about. + String jobName = "JOB_NAME"; + + getJob(projectId, region, jobName); + } + + // Retrieve information about a Batch Job. + public static void getJob(String projectId, String region, String jobName) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the `batchServiceClient.close()` method on the client to safely + // clean up any remaining background resources. + try (BatchServiceClient batchServiceClient = BatchServiceClient.create()) { + + Job job = + batchServiceClient.getJob( + JobName.newBuilder() + .setProject(projectId) + .setLocation(region) + .setJob(jobName) + .build()); + + System.out.printf("Retrieved the job: %s ", job.getName()); + } + } +} +// [END batch_get_job] diff --git a/batch/snippets/src/main/java/ListJobs.java b/batch/snippets/src/main/java/ListJobs.java new file mode 100644 index 00000000000..e58bc80d5db --- /dev/null +++ b/batch/snippets/src/main/java/ListJobs.java @@ -0,0 +1,52 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// [START batch_list_jobs] + +import com.google.cloud.batch.v1.BatchServiceClient; +import com.google.cloud.batch.v1.Job; +import java.io.IOException; + +public class ListJobs { + + public static void main(String[] args) throws IOException { + // TODO(developer): Replace these variables before running the sample. + // Project ID or project number of the Cloud project you want to use. + String projectId = "YOUR_PROJECT_ID"; + + // Name of the region hosting the jobs. + String region = "europe-central2"; + + listJobs(projectId, region); + } + + // Get a list of all jobs defined in given region. + public static void listJobs(String projectId, String region) throws IOException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the `batchServiceClient.close()` method on the client to safely + // clean up any remaining background resources. + try (BatchServiceClient batchServiceClient = BatchServiceClient.create()) { + + // Construct the parent path of the job. + String parent = String.format("projects/%s/locations/%s", projectId, region); + + for (Job job : batchServiceClient.listJobs(parent).iterateAll()) { + System.out.println(job.getName()); + } + System.out.println("Listed all batch jobs."); + } + } +} +// [END batch_list_jobs] diff --git a/batch/snippets/src/test/java/BatchBasicIT.java b/batch/snippets/src/test/java/BatchBasicIT.java new file mode 100644 index 00000000000..98c2a04bf4f --- /dev/null +++ b/batch/snippets/src/test/java/BatchBasicIT.java @@ -0,0 +1,119 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class BatchBasicIT { + + private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); + private static final String REGION = "europe-north1"; + private static String SCRIPT_JOB_NAME; + private static String CONTAINER_JOB_NAME; + + private ByteArrayOutputStream stdOut; + + // Check if the required environment variables are set. + public static void requireEnvVar(String envVarName) { + assertWithMessage(String.format("Missing environment variable '%s' ", envVarName)) + .that(System.getenv(envVarName)) + .isNotEmpty(); + } + + @BeforeClass + public static void setUp() + throws IOException, InterruptedException, ExecutionException, TimeoutException { + final PrintStream out = System.out; + ByteArrayOutputStream stdOut = new ByteArrayOutputStream(); + System.setOut(new PrintStream(stdOut)); + requireEnvVar("GOOGLE_APPLICATION_CREDENTIALS"); + requireEnvVar("GOOGLE_CLOUD_PROJECT"); + + String uuid = String.valueOf(UUID.randomUUID()); + SCRIPT_JOB_NAME = "test-job-script-" + uuid; + CONTAINER_JOB_NAME = "test-job-container-" + uuid; + + CreateWithContainerNoMounting.createContainerJob(PROJECT_ID, REGION, CONTAINER_JOB_NAME); + assertThat(stdOut.toString()) + .contains( + "Successfully created the job: " + + String.format( + "projects/%s/locations/%s/jobs/%s", PROJECT_ID, REGION, CONTAINER_JOB_NAME)); + CreateWithScriptNoMounting.createScriptJob(PROJECT_ID, REGION, SCRIPT_JOB_NAME); + assertThat(stdOut.toString()) + .contains( + "Successfully created the job: " + + String.format( + "projects/%s/locations/%s/jobs/%s", PROJECT_ID, REGION, SCRIPT_JOB_NAME)); + TimeUnit.SECONDS.sleep(10); + + stdOut.close(); + System.setOut(out); + } + + @AfterClass + public static void cleanup() + throws IOException, InterruptedException, ExecutionException, TimeoutException { + final PrintStream out = System.out; + ByteArrayOutputStream stdOut = new ByteArrayOutputStream(); + System.setOut(new PrintStream(stdOut)); + + DeleteJob.deleteJob(PROJECT_ID, REGION, CONTAINER_JOB_NAME); + DeleteJob.deleteJob(PROJECT_ID, REGION, SCRIPT_JOB_NAME); + + stdOut.close(); + System.setOut(out); + } + + @Before + public void beforeEach() { + stdOut = new ByteArrayOutputStream(); + System.setOut(new PrintStream(stdOut)); + } + + @After + public void afterEach() { + stdOut = null; + System.setOut(null); + } + + @Test + public void testGetJob() throws IOException { + GetJob.getJob(PROJECT_ID, REGION, CONTAINER_JOB_NAME); + assertThat(stdOut.toString()).contains("Retrieved the job"); + } + + @Test + public void testListJobs() throws IOException { + ListJobs.listJobs(PROJECT_ID, REGION); + assertThat(stdOut.toString()).contains(CONTAINER_JOB_NAME); + assertThat(stdOut.toString()).contains(SCRIPT_JOB_NAME); + } +}