gcp.dataproc.Batch

import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";

const exampleBatchSpark = new gcp.dataproc.Batch("example_batch_spark", {
    batchId: "tf-test-batch_75125",
    location: "us-central1",
    labels: {
        batch_test: "terraform",
    },
    runtimeConfig: {
        properties: {
            "spark.dynamicAllocation.enabled": "false",
            "spark.executor.instances": "2",
        },
    },
    environmentConfig: {
        executionConfig: {
            subnetworkUri: "default",
            ttl: "3600s",
            networkTags: ["tag1"],
        },
    },
    sparkBatch: {
        mainClass: "org.apache.spark.examples.SparkPi",
        args: ["10"],
        jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
    },
});

import pulumi
import pulumi_gcp as gcp

example_batch_spark = gcp.dataproc.Batch("example_batch_spark",
    batch_id="tf-test-batch_75125",
    location="us-central1",
    labels={
        "batch_test": "terraform",
    },
    runtime_config={
        "properties": {
            "spark.dynamicAllocation.enabled": "false",
            "spark.executor.instances": "2",
        },
    },
    environment_config={
        "execution_config": {
            "subnetwork_uri": "default",
            "ttl": "3600s",
            "network_tags": ["tag1"],
        },
    },
    spark_batch={
        "main_class": "org.apache.spark.examples.SparkPi",
        "args": ["10"],
        "jar_file_uris": ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
    })

package main

import (
	"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/dataproc"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := dataproc.NewBatch(ctx, "example_batch_spark", &dataproc.BatchArgs{
			BatchId:  pulumi.String("tf-test-batch_75125"),
			Location: pulumi.String("us-central1"),
			Labels: pulumi.StringMap{
				"batch_test": pulumi.String("terraform"),
			},
			RuntimeConfig: &dataproc.BatchRuntimeConfigArgs{
				Properties: pulumi.StringMap{
					"spark.dynamicAllocation.enabled": pulumi.String("false"),
					"spark.executor.instances":        pulumi.String("2"),
				},
			},
			EnvironmentConfig: &dataproc.BatchEnvironmentConfigArgs{
				ExecutionConfig: &dataproc.BatchEnvironmentConfigExecutionConfigArgs{
					SubnetworkUri: pulumi.String("default"),
					Ttl:           pulumi.String("3600s"),
					NetworkTags: pulumi.StringArray{
						pulumi.String("tag1"),
					},
				},
			},
			SparkBatch: &dataproc.BatchSparkBatchArgs{
				MainClass: pulumi.String("org.apache.spark.examples.SparkPi"),
				Args: pulumi.StringArray{
					pulumi.String("10"),
				},
				JarFileUris: pulumi.StringArray{
					pulumi.String("file:///usr/lib/spark/examples/jars/spark-examples.jar"),
				},
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;

return await Deployment.RunAsync(() => 
{
    var exampleBatchSpark = new Gcp.Dataproc.Batch("example_batch_spark", new()
    {
        BatchId = "tf-test-batch_75125",
        Location = "us-central1",
        Labels = 
        {
            { "batch_test", "terraform" },
        },
        RuntimeConfig = new Gcp.Dataproc.Inputs.BatchRuntimeConfigArgs
        {
            Properties = 
            {
                { "spark.dynamicAllocation.enabled", "false" },
                { "spark.executor.instances", "2" },
            },
        },
        EnvironmentConfig = new Gcp.Dataproc.Inputs.BatchEnvironmentConfigArgs
        {
            ExecutionConfig = new Gcp.Dataproc.Inputs.BatchEnvironmentConfigExecutionConfigArgs
            {
                SubnetworkUri = "default",
                Ttl = "3600s",
                NetworkTags = new[]
                {
                    "tag1",
                },
            },
        },
        SparkBatch = new Gcp.Dataproc.Inputs.BatchSparkBatchArgs
        {
            MainClass = "org.apache.spark.examples.SparkPi",
            Args = new[]
            {
                "10",
            },
            JarFileUris = new[]
            {
                "file:///usr/lib/spark/examples/jars/spark-examples.jar",
            },
        },
    });

});

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.dataproc.Batch;
import com.pulumi.gcp.dataproc.BatchArgs;
import com.pulumi.gcp.dataproc.inputs.BatchRuntimeConfigArgs;
import com.pulumi.gcp.dataproc.inputs.BatchEnvironmentConfigArgs;
import com.pulumi.gcp.dataproc.inputs.BatchEnvironmentConfigExecutionConfigArgs;
import com.pulumi.gcp.dataproc.inputs.BatchSparkBatchArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var exampleBatchSpark = new Batch("exampleBatchSpark", BatchArgs.builder()
            .batchId("tf-test-batch_75125")
            .location("us-central1")
            .labels(Map.of("batch_test", "terraform"))
            .runtimeConfig(BatchRuntimeConfigArgs.builder()
                .properties(Map.ofEntries(
                    Map.entry("spark.dynamicAllocation.enabled", "false"),
                    Map.entry("spark.executor.instances", "2")
                ))
                .build())
            .environmentConfig(BatchEnvironmentConfigArgs.builder()
                .executionConfig(BatchEnvironmentConfigExecutionConfigArgs.builder()
                    .subnetworkUri("default")
                    .ttl("3600s")
                    .networkTags("tag1")
                    .build())
                .build())
            .sparkBatch(BatchSparkBatchArgs.builder()
                .mainClass("org.apache.spark.examples.SparkPi")
                .args("10")
                .jarFileUris("file:///usr/lib/spark/examples/jars/spark-examples.jar")
                .build())
            .build());

    }
}

resources:
  exampleBatchSpark:
    type: gcp:dataproc:Batch
    name: example_batch_spark
    properties:
      batchId: tf-test-batch_75125
      location: us-central1
      labels:
        batch_test: terraform
      runtimeConfig:
        properties:
          spark.dynamicAllocation.enabled: 'false'
          spark.executor.instances: '2'
      environmentConfig:
        executionConfig:
          subnetworkUri: default
          ttl: 3600s
          networkTags:
            - tag1
      sparkBatch:
        mainClass: org.apache.spark.examples.SparkPi
        args:
          - '10'
        jarFileUris:
          - file:///usr/lib/spark/examples/jars/spark-examples.jar

import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";

const project = gcp.organizations.getProject({});
const gcsAccount = gcp.storage.getProjectServiceAccount({});
const bucket = new gcp.storage.Bucket("bucket", {
    uniformBucketLevelAccess: true,
    name: "dataproc-bucket",
    location: "US",
    forceDestroy: true,
});
const keyRing = new gcp.kms.KeyRing("key_ring", {
    name: "example-keyring",
    location: "us-central1",
});
const cryptoKey = new gcp.kms.CryptoKey("crypto_key", {
    name: "example-key",
    keyRing: keyRing.id,
    purpose: "ENCRYPT_DECRYPT",
});
const cryptoKeyMember1 = new gcp.kms.CryptoKeyIAMMember("crypto_key_member_1", {
    cryptoKeyId: cryptoKey.id,
    role: "roles/cloudkms.cryptoKeyEncrypterDecrypter",
    member: project.then(project => `serviceAccount:service-${project.number}@dataproc-accounts.iam.gserviceaccount.com`),
});
const ms = new gcp.dataproc.MetastoreService("ms", {
    serviceId: "dataproc-batch",
    location: "us-central1",
    port: 9080,
    tier: "DEVELOPER",
    maintenanceWindow: {
        hourOfDay: 2,
        dayOfWeek: "SUNDAY",
    },
    hiveMetastoreConfig: {
        version: "3.1.2",
    },
});
const basic = new gcp.dataproc.Cluster("basic", {
    name: "dataproc-batch",
    region: "us-central1",
    clusterConfig: {
        softwareConfig: {
            overrideProperties: {
                "dataproc:dataproc.allow.zero.workers": "true",
                "spark:spark.history.fs.logDirectory": pulumi.interpolate`gs://${bucket.name}/*/spark-job-history`,
            },
        },
        endpointConfig: {
            enableHttpPortAccess: true,
        },
        masterConfig: {
            numInstances: 1,
            machineType: "e2-standard-2",
            diskConfig: {
                bootDiskSizeGb: 35,
            },
        },
        metastoreConfig: {
            dataprocMetastoreService: ms.name,
        },
    },
});
const exampleBatchSpark = new gcp.dataproc.Batch("example_batch_spark", {
    batchId: "dataproc-batch",
    location: "us-central1",
    labels: {
        batch_test: "terraform",
    },
    runtimeConfig: {
        properties: {
            "spark.dynamicAllocation.enabled": "false",
            "spark.executor.instances": "2",
        },
        version: "2.2",
    },
    environmentConfig: {
        executionConfig: {
            ttl: "3600s",
            networkTags: ["tag1"],
            kmsKey: cryptoKey.id,
            networkUri: "default",
            serviceAccount: project.then(project => `${project.number}-compute@developer.gserviceaccount.com`),
            stagingBucket: bucket.name,
        },
        peripheralsConfig: {
            metastoreService: ms.name,
            sparkHistoryServerConfig: {
                dataprocCluster: basic.id,
            },
        },
    },
    sparkBatch: {
        mainClass: "org.apache.spark.examples.SparkPi",
        args: ["10"],
        jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
    },
}, {
    dependsOn: [cryptoKeyMember1],
});

import pulumi
import pulumi_gcp as gcp

project = gcp.organizations.get_project()
gcs_account = gcp.storage.get_project_service_account()
bucket = gcp.storage.Bucket("bucket",
    uniform_bucket_level_access=True,
    name="dataproc-bucket",
    location="US",
    force_destroy=True)
key_ring = gcp.kms.KeyRing("key_ring",
    name="example-keyring",
    location="us-central1")
crypto_key = gcp.kms.CryptoKey("crypto_key",
    name="example-key",
    key_ring=key_ring.id,
    purpose="ENCRYPT_DECRYPT")
crypto_key_member1 = gcp.kms.CryptoKeyIAMMember("crypto_key_member_1",
    crypto_key_id=crypto_key.id,
    role="roles/cloudkms.cryptoKeyEncrypterDecrypter",
    member=f"serviceAccount:service-{project.number}@dataproc-accounts.iam.gserviceaccount.com")
ms = gcp.dataproc.MetastoreService("ms",
    service_id="dataproc-batch",
    location="us-central1",
    port=9080,
    tier="DEVELOPER",
    maintenance_window={
        "hour_of_day": 2,
        "day_of_week": "SUNDAY",
    },
    hive_metastore_config={
        "version": "3.1.2",
    })
basic = gcp.dataproc.Cluster("basic",
    name="dataproc-batch",
    region="us-central1",
    cluster_config={
        "software_config": {
            "override_properties": {
                "dataproc:dataproc.allow.zero.workers": "true",
                "spark:spark.history.fs.logDirectory": bucket.name.apply(lambda name: f"gs://{name}/*/spark-job-history"),
            },
        },
        "endpoint_config": {
            "enable_http_port_access": True,
        },
        "master_config": {
            "num_instances": 1,
            "machine_type": "e2-standard-2",
            "disk_config": {
                "boot_disk_size_gb": 35,
            },
        },
        "metastore_config": {
            "dataproc_metastore_service": ms.name,
        },
    })
example_batch_spark = gcp.dataproc.Batch("example_batch_spark",
    batch_id="dataproc-batch",
    location="us-central1",
    labels={
        "batch_test": "terraform",
    },
    runtime_config={
        "properties": {
            "spark.dynamicAllocation.enabled": "false",
            "spark.executor.instances": "2",
        },
        "version": "2.2",
    },
    environment_config={
        "execution_config": {
            "ttl": "3600s",
            "network_tags": ["tag1"],
            "kms_key": crypto_key.id,
            "network_uri": "default",
            "service_account": f"{project.number}-compute@developer.gserviceaccount.com",
            "staging_bucket": bucket.name,
        },
        "peripherals_config": {
            "metastore_service": ms.name,
            "spark_history_server_config": {
                "dataproc_cluster": basic.id,
            },
        },
    },
    spark_batch={
        "main_class": "org.apache.spark.examples.SparkPi",
        "args": ["10"],
        "jar_file_uris": ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
    },
    opts = pulumi.ResourceOptions(depends_on=[crypto_key_member1]))

package main

import (
	"fmt"

	"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/dataproc"
	"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/kms"
	"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/organizations"
	"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/storage"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		project, err := organizations.LookupProject(ctx, &organizations.LookupProjectArgs{}, nil)
		if err != nil {
			return err
		}
		_, err = storage.GetProjectServiceAccount(ctx, &storage.GetProjectServiceAccountArgs{}, nil)
		if err != nil {
			return err
		}
		bucket, err := storage.NewBucket(ctx, "bucket", &storage.BucketArgs{
			UniformBucketLevelAccess: pulumi.Bool(true),
			Name:                     pulumi.String("dataproc-bucket"),
			Location:                 pulumi.String("US"),
			ForceDestroy:             pulumi.Bool(true),
		})
		if err != nil {
			return err
		}
		keyRing, err := kms.NewKeyRing(ctx, "key_ring", &kms.KeyRingArgs{
			Name:     pulumi.String("example-keyring"),
			Location: pulumi.String("us-central1"),
		})
		if err != nil {
			return err
		}
		cryptoKey, err := kms.NewCryptoKey(ctx, "crypto_key", &kms.CryptoKeyArgs{
			Name:    pulumi.String("example-key"),
			KeyRing: keyRing.ID(),
			Purpose: pulumi.String("ENCRYPT_DECRYPT"),
		})
		if err != nil {
			return err
		}
		cryptoKeyMember1, err := kms.NewCryptoKeyIAMMember(ctx, "crypto_key_member_1", &kms.CryptoKeyIAMMemberArgs{
			CryptoKeyId: cryptoKey.ID(),
			Role:        pulumi.String("roles/cloudkms.cryptoKeyEncrypterDecrypter"),
			Member:      pulumi.Sprintf("serviceAccount:service-%v@dataproc-accounts.iam.gserviceaccount.com", project.Number),
		})
		if err != nil {
			return err
		}
		ms, err := dataproc.NewMetastoreService(ctx, "ms", &dataproc.MetastoreServiceArgs{
			ServiceId: pulumi.String("dataproc-batch"),
			Location:  pulumi.String("us-central1"),
			Port:      pulumi.Int(9080),
			Tier:      pulumi.String("DEVELOPER"),
			MaintenanceWindow: &dataproc.MetastoreServiceMaintenanceWindowArgs{
				HourOfDay: pulumi.Int(2),
				DayOfWeek: pulumi.String("SUNDAY"),
			},
			HiveMetastoreConfig: &dataproc.MetastoreServiceHiveMetastoreConfigArgs{
				Version: pulumi.String("3.1.2"),
			},
		})
		if err != nil {
			return err
		}
		basic, err := dataproc.NewCluster(ctx, "basic", &dataproc.ClusterArgs{
			Name:   pulumi.String("dataproc-batch"),
			Region: pulumi.String("us-central1"),
			ClusterConfig: &dataproc.ClusterClusterConfigArgs{
				SoftwareConfig: &dataproc.ClusterClusterConfigSoftwareConfigArgs{
					OverrideProperties: pulumi.StringMap{
						"dataproc:dataproc.allow.zero.workers": pulumi.String("true"),
						"spark:spark.history.fs.logDirectory": bucket.Name.ApplyT(func(name string) (string, error) {
							return fmt.Sprintf("gs://%v/*/spark-job-history", name), nil
						}).(pulumi.StringOutput),
					},
				},
				EndpointConfig: &dataproc.ClusterClusterConfigEndpointConfigArgs{
					EnableHttpPortAccess: pulumi.Bool(true),
				},
				MasterConfig: &dataproc.ClusterClusterConfigMasterConfigArgs{
					NumInstances: pulumi.Int(1),
					MachineType:  pulumi.String("e2-standard-2"),
					DiskConfig: &dataproc.ClusterClusterConfigMasterConfigDiskConfigArgs{
						BootDiskSizeGb: pulumi.Int(35),
					},
				},
				MetastoreConfig: &dataproc.ClusterClusterConfigMetastoreConfigArgs{
					DataprocMetastoreService: ms.Name,
				},
			},
		})
		if err != nil {
			return err
		}
		_, err = dataproc.NewBatch(ctx, "example_batch_spark", &dataproc.BatchArgs{
			BatchId:  pulumi.String("dataproc-batch"),
			Location: pulumi.String("us-central1"),
			Labels: pulumi.StringMap{
				"batch_test": pulumi.String("terraform"),
			},
			RuntimeConfig: &dataproc.BatchRuntimeConfigArgs{
				Properties: pulumi.StringMap{
					"spark.dynamicAllocation.enabled": pulumi.String("false"),
					"spark.executor.instances":        pulumi.String("2"),
				},
				Version: pulumi.String("2.2"),
			},
			EnvironmentConfig: &dataproc.BatchEnvironmentConfigArgs{
				ExecutionConfig: &dataproc.BatchEnvironmentConfigExecutionConfigArgs{
					Ttl: pulumi.String("3600s"),
					NetworkTags: pulumi.StringArray{
						pulumi.String("tag1"),
					},
					KmsKey:         cryptoKey.ID(),
					NetworkUri:     pulumi.String("default"),
					ServiceAccount: pulumi.Sprintf("%v-compute@developer.gserviceaccount.com", project.Number),
					StagingBucket:  bucket.Name,
				},
				PeripheralsConfig: &dataproc.BatchEnvironmentConfigPeripheralsConfigArgs{
					MetastoreService: ms.Name,
					SparkHistoryServerConfig: &dataproc.BatchEnvironmentConfigPeripheralsConfigSparkHistoryServerConfigArgs{
						DataprocCluster: basic.ID(),
					},
				},
			},
			SparkBatch: &dataproc.BatchSparkBatchArgs{
				MainClass: pulumi.String("org.apache.spark.examples.SparkPi"),
				Args: pulumi.StringArray{
					pulumi.String("10"),
				},
				JarFileUris: pulumi.StringArray{
					pulumi.String("file:///usr/lib/spark/examples/jars/spark-examples.jar"),
				},
			},
		}, pulumi.DependsOn([]pulumi.Resource{
			cryptoKeyMember1,
		}))
		if err != nil {
			return err
		}
		return nil
	})
}

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;

return await Deployment.RunAsync(() => 
{
    var project = Gcp.Organizations.GetProject.Invoke();

    var gcsAccount = Gcp.Storage.GetProjectServiceAccount.Invoke();

    var bucket = new Gcp.Storage.Bucket("bucket", new()
    {
        UniformBucketLevelAccess = true,
        Name = "dataproc-bucket",
        Location = "US",
        ForceDestroy = true,
    });

    var keyRing = new Gcp.Kms.KeyRing("key_ring", new()
    {
        Name = "example-keyring",
        Location = "us-central1",
    });

    var cryptoKey = new Gcp.Kms.CryptoKey("crypto_key", new()
    {
        Name = "example-key",
        KeyRing = keyRing.Id,
        Purpose = "ENCRYPT_DECRYPT",
    });

    var cryptoKeyMember1 = new Gcp.Kms.CryptoKeyIAMMember("crypto_key_member_1", new()
    {
        CryptoKeyId = cryptoKey.Id,
        Role = "roles/cloudkms.cryptoKeyEncrypterDecrypter",
        Member = $"serviceAccount:service-{project.Apply(getProjectResult => getProjectResult.Number)}@dataproc-accounts.iam.gserviceaccount.com",
    });

    var ms = new Gcp.Dataproc.MetastoreService("ms", new()
    {
        ServiceId = "dataproc-batch",
        Location = "us-central1",
        Port = 9080,
        Tier = "DEVELOPER",
        MaintenanceWindow = new Gcp.Dataproc.Inputs.MetastoreServiceMaintenanceWindowArgs
        {
            HourOfDay = 2,
            DayOfWeek = "SUNDAY",
        },
        HiveMetastoreConfig = new Gcp.Dataproc.Inputs.MetastoreServiceHiveMetastoreConfigArgs
        {
            Version = "3.1.2",
        },
    });

    var basic = new Gcp.Dataproc.Cluster("basic", new()
    {
        Name = "dataproc-batch",
        Region = "us-central1",
        ClusterConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigArgs
        {
            SoftwareConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigSoftwareConfigArgs
            {
                OverrideProperties = 
                {
                    { "dataproc:dataproc.allow.zero.workers", "true" },
                    { "spark:spark.history.fs.logDirectory", bucket.Name.Apply(name => $"gs://{name}/*/spark-job-history") },
                },
            },
            EndpointConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigEndpointConfigArgs
            {
                EnableHttpPortAccess = true,
            },
            MasterConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigMasterConfigArgs
            {
                NumInstances = 1,
                MachineType = "e2-standard-2",
                DiskConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigMasterConfigDiskConfigArgs
                {
                    BootDiskSizeGb = 35,
                },
            },
            MetastoreConfig = new Gcp.Dataproc.Inputs.ClusterClusterConfigMetastoreConfigArgs
            {
                DataprocMetastoreService = ms.Name,
            },
        },
    });

    var exampleBatchSpark = new Gcp.Dataproc.Batch("example_batch_spark", new()
    {
        BatchId = "dataproc-batch",
        Location = "us-central1",
        Labels = 
        {
            { "batch_test", "terraform" },
        },
        RuntimeConfig = new Gcp.Dataproc.Inputs.BatchRuntimeConfigArgs
        {
            Properties = 
            {
                { "spark.dynamicAllocation.enabled", "false" },
                { "spark.executor.instances", "2" },
            },
            Version = "2.2",
        },
        EnvironmentConfig = new Gcp.Dataproc.Inputs.BatchEnvironmentConfigArgs
        {
            ExecutionConfig = new Gcp.Dataproc.Inputs.BatchEnvironmentConfigExecutionConfigArgs
            {
                Ttl = "3600s",
                NetworkTags = new[]
                {
                    "tag1",
                },
                KmsKey = cryptoKey.Id,
                NetworkUri = "default",
                ServiceAccount = $"{project.Apply(getProjectResult => getProjectResult.Number)}-compute@developer.gserviceaccount.com",
                StagingBucket = bucket.Name,
            },
            PeripheralsConfig = new Gcp.Dataproc.Inputs.BatchEnvironmentConfigPeripheralsConfigArgs
            {
                MetastoreService = ms.Name,
                SparkHistoryServerConfig = new Gcp.Dataproc.Inputs.BatchEnvironmentConfigPeripheralsConfigSparkHistoryServerConfigArgs
                {
                    DataprocCluster = basic.Id,
                },
            },
        },
        SparkBatch = new Gcp.Dataproc.Inputs.BatchSparkBatchArgs
        {
            MainClass = "org.apache.spark.examples.SparkPi",
            Args = new[]
            {
                "10",
            },
            JarFileUris = new[]
            {
                "file:///usr/lib/spark/examples/jars/spark-examples.jar",
            },
        },
    }, new CustomResourceOptions
    {
        DependsOn =
        {
            cryptoKeyMember1,
        },
    });

});

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.organizations.OrganizationsFunctions;
import com.pulumi.gcp.organizations.inputs.GetProjectArgs;
import com.pulumi.gcp.storage.StorageFunctions;
import com.pulumi.gcp.storage.inputs.GetProjectServiceAccountArgs;
import com.pulumi.gcp.storage.Bucket;
import com.pulumi.gcp.storage.BucketArgs;
import com.pulumi.gcp.kms.KeyRing;
import com.pulumi.gcp.kms.KeyRingArgs;
import com.pulumi.gcp.kms.CryptoKey;
import com.pulumi.gcp.kms.CryptoKeyArgs;
import com.pulumi.gcp.kms.CryptoKeyIAMMember;
import com.pulumi.gcp.kms.CryptoKeyIAMMemberArgs;
import com.pulumi.gcp.dataproc.MetastoreService;
import com.pulumi.gcp.dataproc.MetastoreServiceArgs;
import com.pulumi.gcp.dataproc.inputs.MetastoreServiceMaintenanceWindowArgs;
import com.pulumi.gcp.dataproc.inputs.MetastoreServiceHiveMetastoreConfigArgs;
import com.pulumi.gcp.dataproc.Cluster;
import com.pulumi.gcp.dataproc.ClusterArgs;
import com.pulumi.gcp.dataproc.inputs.ClusterClusterConfigArgs;
import com.pulumi.gcp.dataproc.inputs.ClusterClusterConfigSoftwareConfigArgs;
import com.pulumi.gcp.dataproc.inputs.ClusterClusterConfigEndpointConfigArgs;
import com.pulumi.gcp.dataproc.inputs.ClusterClusterConfigMasterConfigArgs;
import com.pulumi.gcp.dataproc.inputs.ClusterClusterConfigMasterConfigDiskConfigArgs;
import com.pulumi.gcp.dataproc.inputs.ClusterClusterConfigMetastoreConfigArgs;
import com.pulumi.gcp.dataproc.Batch;
import com.pulumi.gcp.dataproc.BatchArgs;
import com.pulumi.gcp.dataproc.inputs.BatchRuntimeConfigArgs;
import com.pulumi.gcp.dataproc.inputs.BatchEnvironmentConfigArgs;
import com.pulumi.gcp.dataproc.inputs.BatchEnvironmentConfigExecutionConfigArgs;
import com.pulumi.gcp.dataproc.inputs.BatchEnvironmentConfigPeripheralsConfigArgs;
import com.pulumi.gcp.dataproc.inputs.BatchEnvironmentConfigPeripheralsConfigSparkHistoryServerConfigArgs;
import com.pulumi.gcp.dataproc.inputs.BatchSparkBatchArgs;
import com.pulumi.resources.CustomResourceOptions;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        final var project = OrganizationsFunctions.getProject();

        final var gcsAccount = StorageFunctions.getProjectServiceAccount();

        var bucket = new Bucket("bucket", BucketArgs.builder()
            .uniformBucketLevelAccess(true)
            .name("dataproc-bucket")
            .location("US")
            .forceDestroy(true)
            .build());

        var keyRing = new KeyRing("keyRing", KeyRingArgs.builder()
            .name("example-keyring")
            .location("us-central1")
            .build());

        var cryptoKey = new CryptoKey("cryptoKey", CryptoKeyArgs.builder()
            .name("example-key")
            .keyRing(keyRing.id())
            .purpose("ENCRYPT_DECRYPT")
            .build());

        var cryptoKeyMember1 = new CryptoKeyIAMMember("cryptoKeyMember1", CryptoKeyIAMMemberArgs.builder()
            .cryptoKeyId(cryptoKey.id())
            .role("roles/cloudkms.cryptoKeyEncrypterDecrypter")
            .member(String.format("serviceAccount:service-%s@dataproc-accounts.iam.gserviceaccount.com", project.applyValue(getProjectResult -> getProjectResult.number())))
            .build());

        var ms = new MetastoreService("ms", MetastoreServiceArgs.builder()
            .serviceId("dataproc-batch")
            .location("us-central1")
            .port(9080)
            .tier("DEVELOPER")
            .maintenanceWindow(MetastoreServiceMaintenanceWindowArgs.builder()
                .hourOfDay(2)
                .dayOfWeek("SUNDAY")
                .build())
            .hiveMetastoreConfig(MetastoreServiceHiveMetastoreConfigArgs.builder()
                .version("3.1.2")
                .build())
            .build());

        var basic = new Cluster("basic", ClusterArgs.builder()
            .name("dataproc-batch")
            .region("us-central1")
            .clusterConfig(ClusterClusterConfigArgs.builder()
                .softwareConfig(ClusterClusterConfigSoftwareConfigArgs.builder()
                    .overrideProperties(Map.ofEntries(
                        Map.entry("dataproc:dataproc.allow.zero.workers", "true"),
                        Map.entry("spark:spark.history.fs.logDirectory", bucket.name().applyValue(name -> String.format("gs://%s/*/spark-job-history", name)))
                    ))
                    .build())
                .endpointConfig(ClusterClusterConfigEndpointConfigArgs.builder()
                    .enableHttpPortAccess(true)
                    .build())
                .masterConfig(ClusterClusterConfigMasterConfigArgs.builder()
                    .numInstances(1)
                    .machineType("e2-standard-2")
                    .diskConfig(ClusterClusterConfigMasterConfigDiskConfigArgs.builder()
                        .bootDiskSizeGb(35)
                        .build())
                    .build())
                .metastoreConfig(ClusterClusterConfigMetastoreConfigArgs.builder()
                    .dataprocMetastoreService(ms.name())
                    .build())
                .build())
            .build());

        var exampleBatchSpark = new Batch("exampleBatchSpark", BatchArgs.builder()
            .batchId("dataproc-batch")
            .location("us-central1")
            .labels(Map.of("batch_test", "terraform"))
            .runtimeConfig(BatchRuntimeConfigArgs.builder()
                .properties(Map.ofEntries(
                    Map.entry("spark.dynamicAllocation.enabled", "false"),
                    Map.entry("spark.executor.instances", "2")
                ))
                .version("2.2")
                .build())
            .environmentConfig(BatchEnvironmentConfigArgs.builder()
                .executionConfig(BatchEnvironmentConfigExecutionConfigArgs.builder()
                    .ttl("3600s")
                    .networkTags("tag1")
                    .kmsKey(cryptoKey.id())
                    .networkUri("default")
                    .serviceAccount(String.format("%s-compute@developer.gserviceaccount.com", project.applyValue(getProjectResult -> getProjectResult.number())))
                    .stagingBucket(bucket.name())
                    .build())
                .peripheralsConfig(BatchEnvironmentConfigPeripheralsConfigArgs.builder()
                    .metastoreService(ms.name())
                    .sparkHistoryServerConfig(BatchEnvironmentConfigPeripheralsConfigSparkHistoryServerConfigArgs.builder()
                        .dataprocCluster(basic.id())
                        .build())
                    .build())
                .build())
            .sparkBatch(BatchSparkBatchArgs.builder()
                .mainClass("org.apache.spark.examples.SparkPi")
                .args("10")
                .jarFileUris("file:///usr/lib/spark/examples/jars/spark-examples.jar")
                .build())
            .build(), CustomResourceOptions.builder()
                .dependsOn(cryptoKeyMember1)
                .build());

    }
}

resources:
  exampleBatchSpark:
    type: gcp:dataproc:Batch
    name: example_batch_spark
    properties:
      batchId: dataproc-batch
      location: us-central1
      labels:
        batch_test: terraform
      runtimeConfig:
        properties:
          spark.dynamicAllocation.enabled: 'false'
          spark.executor.instances: '2'
        version: '2.2'
      environmentConfig:
        executionConfig:
          ttl: 3600s
          networkTags:
            - tag1
          kmsKey: ${cryptoKey.id}
          networkUri: default
          serviceAccount: ${project.number}-compute@developer.gserviceaccount.com
          stagingBucket: ${bucket.name}
        peripheralsConfig:
          metastoreService: ${ms.name}
          sparkHistoryServerConfig:
            dataprocCluster: ${basic.id}
      sparkBatch:
        mainClass: org.apache.spark.examples.SparkPi
        args:
          - '10'
        jarFileUris:
          - file:///usr/lib/spark/examples/jars/spark-examples.jar
    options:
      dependson:
        - ${cryptoKeyMember1}
  bucket:
    type: gcp:storage:Bucket
    properties:
      uniformBucketLevelAccess: true
      name: dataproc-bucket
      location: US
      forceDestroy: true
  cryptoKey:
    type: gcp:kms:CryptoKey
    name: crypto_key
    properties:
      name: example-key
      keyRing: ${keyRing.id}
      purpose: ENCRYPT_DECRYPT
  keyRing:
    type: gcp:kms:KeyRing
    name: key_ring
    properties:
      name: example-keyring
      location: us-central1
  cryptoKeyMember1:
    type: gcp:kms:CryptoKeyIAMMember
    name: crypto_key_member_1
    properties:
      cryptoKeyId: ${cryptoKey.id}
      role: roles/cloudkms.cryptoKeyEncrypterDecrypter
      member: serviceAccount:service-${project.number}@dataproc-accounts.iam.gserviceaccount.com
  basic:
    type: gcp:dataproc:Cluster
    properties:
      name: dataproc-batch
      region: us-central1
      clusterConfig:
        softwareConfig:
          overrideProperties:
            dataproc:dataproc.allow.zero.workers: 'true'
            spark:spark.history.fs.logDirectory: gs://${bucket.name}/*/spark-job-history
        endpointConfig:
          enableHttpPortAccess: true
        masterConfig:
          numInstances: 1
          machineType: e2-standard-2
          diskConfig:
            bootDiskSizeGb: 35
        metastoreConfig:
          dataprocMetastoreService: ${ms.name}
  ms:
    type: gcp:dataproc:MetastoreService
    properties:
      serviceId: dataproc-batch
      location: us-central1
      port: 9080
      tier: DEVELOPER
      maintenanceWindow:
        hourOfDay: 2
        dayOfWeek: SUNDAY
      hiveMetastoreConfig:
        version: 3.1.2
variables:
  project:
    fn::invoke:
      Function: gcp:organizations:getProject
      Arguments: {}
  gcsAccount:
    fn::invoke:
      Function: gcp:storage:getProjectServiceAccount
      Arguments: {}

import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";

const exampleBatchSparsql = new gcp.dataproc.Batch("example_batch_sparsql", {
    batchId: "tf-test-batch_88722",
    location: "us-central1",
    runtimeConfig: {
        properties: {
            "spark.dynamicAllocation.enabled": "false",
            "spark.executor.instances": "2",
        },
    },
    environmentConfig: {
        executionConfig: {
            subnetworkUri: "default",
        },
    },
    sparkSqlBatch: {
        queryFileUri: "gs://dataproc-examples/spark-sql/natality/cigarette_correlations.sql",
        jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
        queryVariables: {
            name: "value",
        },
    },
});

import pulumi
import pulumi_gcp as gcp

example_batch_sparsql = gcp.dataproc.Batch("example_batch_sparsql",
    batch_id="tf-test-batch_88722",
    location="us-central1",
    runtime_config={
        "properties": {
            "spark.dynamicAllocation.enabled": "false",
            "spark.executor.instances": "2",
        },
    },
    environment_config={
        "execution_config": {
            "subnetwork_uri": "default",
        },
    },
    spark_sql_batch={
        "query_file_uri": "gs://dataproc-examples/spark-sql/natality/cigarette_correlations.sql",
        "jar_file_uris": ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
        "query_variables": {
            "name": "value",
        },
    })

package main

import (
	"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/dataproc"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := dataproc.NewBatch(ctx, "example_batch_sparsql", &dataproc.BatchArgs{
			BatchId:  pulumi.String("tf-test-batch_88722"),
			Location: pulumi.String("us-central1"),
			RuntimeConfig: &dataproc.BatchRuntimeConfigArgs{
				Properties: pulumi.StringMap{
					"spark.dynamicAllocation.enabled": pulumi.String("false"),
					"spark.executor.instances":        pulumi.String("2"),
				},
			},
			EnvironmentConfig: &dataproc.BatchEnvironmentConfigArgs{
				ExecutionConfig: &dataproc.BatchEnvironmentConfigExecutionConfigArgs{
					SubnetworkUri: pulumi.String("default"),
				},
			},
			SparkSqlBatch: &dataproc.BatchSparkSqlBatchArgs{
				QueryFileUri: pulumi.String("gs://dataproc-examples/spark-sql/natality/cigarette_correlations.sql"),
				JarFileUris: pulumi.StringArray{
					pulumi.String("file:///usr/lib/spark/examples/jars/spark-examples.jar"),
				},
				QueryVariables: pulumi.StringMap{
					"name": pulumi.String("value"),
				},
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;

return await Deployment.RunAsync(() => 
{
    var exampleBatchSparsql = new Gcp.Dataproc.Batch("example_batch_sparsql", new()
    {
        BatchId = "tf-test-batch_88722",
        Location = "us-central1",
        RuntimeConfig = new Gcp.Dataproc.Inputs.BatchRuntimeConfigArgs
        {
            Properties = 
            {
                { "spark.dynamicAllocation.enabled", "false" },
                { "spark.executor.instances", "2" },
            },
        },
        EnvironmentConfig = new Gcp.Dataproc.Inputs.BatchEnvironmentConfigArgs
        {
            ExecutionConfig = new Gcp.Dataproc.Inputs.BatchEnvironmentConfigExecutionConfigArgs
            {
                SubnetworkUri = "default",
            },
        },
        SparkSqlBatch = new Gcp.Dataproc.Inputs.BatchSparkSqlBatchArgs
        {
            QueryFileUri = "gs://dataproc-examples/spark-sql/natality/cigarette_correlations.sql",
            JarFileUris = new[]
            {
                "file:///usr/lib/spark/examples/jars/spark-examples.jar",
            },
            QueryVariables = 
            {
                { "name", "value" },
            },
        },
    });

});

package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.dataproc.Batch;
import com.pulumi.gcp.dataproc.BatchArgs;
import com.pulumi.gcp.dataproc.inputs.BatchRuntimeConfigArgs;
import com.pulumi.gcp.dataproc.inputs.BatchEnvironmentConfigArgs;
import com.pulumi.gcp.dataproc.inputs.BatchEnvironmentConfigExecutionConfigArgs;
import com.pulumi.gcp.dataproc.inputs.BatchSparkSqlBatchArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var exampleBatchSparsql = new Batch("exampleBatchSparsql", BatchArgs.builder()
            .batchId("tf-test-batch_88722")
            .location("us-central1")
            .runtimeConfig(BatchRuntimeConfigArgs.builder()
                .properties(Map.ofEntries(
                    Map.entry("spark.dynamicAllocation.enabled", "false"),
                    Map.entry("spark.executor.instances", "2")
                ))
                .build())
            .environmentConfig(BatchEnvironmentConfigArgs.builder()
                .executionConfig(BatchEnvironmentConfigExecutionConfigArgs.builder()
                    .subnetworkUri("default")
                    .build())
                .build())
            .sparkSqlBatch(BatchSparkSqlBatchArgs.builder()
                .queryFileUri("gs://dataproc-examples/spark-sql/natality/cigarette_correlations.sql")
                .jarFileUris("file:///usr/lib/spark/examples/jars/spark-examples.jar")
                .queryVariables(Map.of("name", "value"))
                .build())
            .build());

    }
}