yandex.DataprocCluster

using System.IO;
using Pulumi;
using Yandex = Pulumi.Yandex;

class MyStack : Stack
{
    public MyStack()
    {
        var fooVpcNetwork = new Yandex.VpcNetwork("fooVpcNetwork", new Yandex.VpcNetworkArgs
        {
        });
        var fooVpcSubnet = new Yandex.VpcSubnet("fooVpcSubnet", new Yandex.VpcSubnetArgs
        {
            Zone = "ru-central1-b",
            NetworkId = fooVpcNetwork.Id,
            V4CidrBlocks = 
            {
                "10.1.0.0/24",
            },
        });
        var dataprocIamServiceAccount = new Yandex.IamServiceAccount("dataprocIamServiceAccount", new Yandex.IamServiceAccountArgs
        {
            Description = "service account to manage Dataproc Cluster",
        });
        var fooResourcemanagerFolder = Output.Create(Yandex.GetResourcemanagerFolder.InvokeAsync(new Yandex.GetResourcemanagerFolderArgs
        {
            FolderId = "some_folder_id",
        }));
        var dataprocResourcemanagerFolderIamBinding = new Yandex.ResourcemanagerFolderIamBinding("dataprocResourcemanagerFolderIamBinding", new Yandex.ResourcemanagerFolderIamBindingArgs
        {
            FolderId = fooResourcemanagerFolder.Apply(fooResourcemanagerFolder => fooResourcemanagerFolder.Id),
            Role = "mdb.dataproc.agent",
            Members = 
            {
                dataprocIamServiceAccount.Id.Apply(id => $"serviceAccount:{id}"),
            },
        });
        // required in order to create bucket
        var bucket_creator = new Yandex.ResourcemanagerFolderIamBinding("bucket-creator", new Yandex.ResourcemanagerFolderIamBindingArgs
        {
            FolderId = fooResourcemanagerFolder.Apply(fooResourcemanagerFolder => fooResourcemanagerFolder.Id),
            Role = "editor",
            Members = 
            {
                dataprocIamServiceAccount.Id.Apply(id => $"serviceAccount:{id}"),
            },
        });
        var fooIamServiceAccountStaticAccessKey = new Yandex.IamServiceAccountStaticAccessKey("fooIamServiceAccountStaticAccessKey", new Yandex.IamServiceAccountStaticAccessKeyArgs
        {
            ServiceAccountId = dataprocIamServiceAccount.Id,
        });
        var fooStorageBucket = new Yandex.StorageBucket("fooStorageBucket", new Yandex.StorageBucketArgs
        {
            Bucket = "foo",
            AccessKey = fooIamServiceAccountStaticAccessKey.AccessKey,
            SecretKey = fooIamServiceAccountStaticAccessKey.SecretKey,
        }, new CustomResourceOptions
        {
            DependsOn = 
            {
                bucket_creator,
            },
        });
        var fooDataprocCluster = new Yandex.DataprocCluster("fooDataprocCluster", new Yandex.DataprocClusterArgs
        {
            Bucket = fooStorageBucket.Bucket,
            Description = "Dataproc Cluster created by Terraform",
            Labels = 
            {
                { "created_by", "terraform" },
            },
            ServiceAccountId = dataprocIamServiceAccount.Id,
            ZoneId = "ru-central1-b",
            ClusterConfig = new Yandex.Inputs.DataprocClusterClusterConfigArgs
            {
                Hadoop = new Yandex.Inputs.DataprocClusterClusterConfigHadoopArgs
                {
                    Services = 
                    {
                        "HDFS",
                        "YARN",
                        "SPARK",
                        "TEZ",
                        "MAPREDUCE",
                        "HIVE",
                    },
                    Properties = 
                    {
                        { "yarn:yarn.resourcemanager.am.max-attempts", "5" },
                    },
                    SshPublicKeys = 
                    {
                        File.ReadAllText("~/.ssh/id_rsa.pub"),
                    },
                },
                SubclusterSpecs = 
                {
                    new Yandex.Inputs.DataprocClusterClusterConfigSubclusterSpecArgs
                    {
                        Name = "main",
                        Role = "MASTERNODE",
                        Resources = new Yandex.Inputs.DataprocClusterClusterConfigSubclusterSpecResourcesArgs
                        {
                            ResourcePresetId = "s2.small",
                            DiskTypeId = "network-hdd",
                            DiskSize = 20,
                        },
                        SubnetId = fooVpcSubnet.Id,
                        HostsCount = 1,
                    },
                    new Yandex.Inputs.DataprocClusterClusterConfigSubclusterSpecArgs
                    {
                        Name = "data",
                        Role = "DATANODE",
                        Resources = new Yandex.Inputs.DataprocClusterClusterConfigSubclusterSpecResourcesArgs
                        {
                            ResourcePresetId = "s2.small",
                            DiskTypeId = "network-hdd",
                            DiskSize = 20,
                        },
                        SubnetId = fooVpcSubnet.Id,
                        HostsCount = 2,
                    },
                    new Yandex.Inputs.DataprocClusterClusterConfigSubclusterSpecArgs
                    {
                        Name = "compute",
                        Role = "COMPUTENODE",
                        Resources = new Yandex.Inputs.DataprocClusterClusterConfigSubclusterSpecResourcesArgs
                        {
                            ResourcePresetId = "s2.small",
                            DiskTypeId = "network-hdd",
                            DiskSize = 20,
                        },
                        SubnetId = fooVpcSubnet.Id,
                        HostsCount = 2,
                    },
                    new Yandex.Inputs.DataprocClusterClusterConfigSubclusterSpecArgs
                    {
                        Name = "compute_autoscaling",
                        Role = "COMPUTENODE",
                        Resources = new Yandex.Inputs.DataprocClusterClusterConfigSubclusterSpecResourcesArgs
                        {
                            ResourcePresetId = "s2.small",
                            DiskTypeId = "network-hdd",
                            DiskSize = 20,
                        },
                        SubnetId = fooVpcSubnet.Id,
                        HostsCount = 2,
                        AutoscalingConfig = new Yandex.Inputs.DataprocClusterClusterConfigSubclusterSpecAutoscalingConfigArgs
                        {
                            MaxHostsCount = 10,
                            MeasurementDuration = 60,
                            WarmupDuration = 60,
                            StabilizationDuration = 120,
                            Preemptible = false,
                            DecommissionTimeout = 60,
                        },
                    },
                },
            },
        }, new CustomResourceOptions
        {
            DependsOn = 
            {
                dataprocResourcemanagerFolderIamBinding,
            },
        });
    }

}

package main

import (
	"fmt"
	"io/ioutil"

	"github.com/pulumi/pulumi-yandex/sdk/go/yandex"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func readFileOrPanic(path string) pulumi.StringPtrInput {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err.Error())
	}
	return pulumi.String(string(data))
}

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		fooVpcNetwork, err := yandex.NewVpcNetwork(ctx, "fooVpcNetwork", nil)
		if err != nil {
			return err
		}
		fooVpcSubnet, err := yandex.NewVpcSubnet(ctx, "fooVpcSubnet", &yandex.VpcSubnetArgs{
			Zone:      pulumi.String("ru-central1-b"),
			NetworkId: fooVpcNetwork.ID(),
			V4CidrBlocks: pulumi.StringArray{
				pulumi.String("10.1.0.0/24"),
			},
		})
		if err != nil {
			return err
		}
		dataprocIamServiceAccount, err := yandex.NewIamServiceAccount(ctx, "dataprocIamServiceAccount", &yandex.IamServiceAccountArgs{
			Description: pulumi.String("service account to manage Dataproc Cluster"),
		})
		if err != nil {
			return err
		}
		opt0 := "some_folder_id"
		fooResourcemanagerFolder, err := yandex.LookupResourcemanagerFolder(ctx, &GetResourcemanagerFolderArgs{
			FolderId: &opt0,
		}, nil)
		if err != nil {
			return err
		}
		dataprocResourcemanagerFolderIamBinding, err := yandex.NewResourcemanagerFolderIamBinding(ctx, "dataprocResourcemanagerFolderIamBinding", &yandex.ResourcemanagerFolderIamBindingArgs{
			FolderId: pulumi.String(fooResourcemanagerFolder.Id),
			Role:     pulumi.String("mdb.dataproc.agent"),
			Members: pulumi.StringArray{
				dataprocIamServiceAccount.ID().ApplyT(func(id string) (string, error) {
					return fmt.Sprintf("%v%v", "serviceAccount:", id), nil
				}).(pulumi.StringOutput),
			},
		})
		if err != nil {
			return err
		}
		_, err = yandex.NewResourcemanagerFolderIamBinding(ctx, "bucket-creator", &yandex.ResourcemanagerFolderIamBindingArgs{
			FolderId: pulumi.String(fooResourcemanagerFolder.Id),
			Role:     pulumi.String("editor"),
			Members: pulumi.StringArray{
				dataprocIamServiceAccount.ID().ApplyT(func(id string) (string, error) {
					return fmt.Sprintf("%v%v", "serviceAccount:", id), nil
				}).(pulumi.StringOutput),
			},
		})
		if err != nil {
			return err
		}
		fooIamServiceAccountStaticAccessKey, err := yandex.NewIamServiceAccountStaticAccessKey(ctx, "fooIamServiceAccountStaticAccessKey", &yandex.IamServiceAccountStaticAccessKeyArgs{
			ServiceAccountId: dataprocIamServiceAccount.ID(),
		})
		if err != nil {
			return err
		}
		fooStorageBucket, err := yandex.NewStorageBucket(ctx, "fooStorageBucket", &yandex.StorageBucketArgs{
			Bucket:    pulumi.String("foo"),
			AccessKey: fooIamServiceAccountStaticAccessKey.AccessKey,
			SecretKey: fooIamServiceAccountStaticAccessKey.SecretKey,
		}, pulumi.DependsOn([]pulumi.Resource{
			bucket_creator,
		}))
		if err != nil {
			return err
		}
		_, err = yandex.NewDataprocCluster(ctx, "fooDataprocCluster", &yandex.DataprocClusterArgs{
			Bucket:      fooStorageBucket.Bucket,
			Description: pulumi.String("Dataproc Cluster created by Terraform"),
			Labels: pulumi.StringMap{
				"created_by": pulumi.String("terraform"),
			},
			ServiceAccountId: dataprocIamServiceAccount.ID(),
			ZoneId:           pulumi.String("ru-central1-b"),
			ClusterConfig: &DataprocClusterClusterConfigArgs{
				Hadoop: &DataprocClusterClusterConfigHadoopArgs{
					Services: pulumi.StringArray{
						pulumi.String("HDFS"),
						pulumi.String("YARN"),
						pulumi.String("SPARK"),
						pulumi.String("TEZ"),
						pulumi.String("MAPREDUCE"),
						pulumi.String("HIVE"),
					},
					Properties: pulumi.StringMap{
						"yarn:yarn.resourcemanager.am.max-attempts": pulumi.String("5"),
					},
					SshPublicKeys: pulumi.StringArray{
						readFileOrPanic("~/.ssh/id_rsa.pub"),
					},
				},
				SubclusterSpecs: DataprocClusterClusterConfigSubclusterSpecArray{
					&DataprocClusterClusterConfigSubclusterSpecArgs{
						Name: pulumi.String("main"),
						Role: pulumi.String("MASTERNODE"),
						Resources: &DataprocClusterClusterConfigSubclusterSpecResourcesArgs{
							ResourcePresetId: pulumi.String("s2.small"),
							DiskTypeId:       pulumi.String("network-hdd"),
							DiskSize:         pulumi.Int(20),
						},
						SubnetId:   fooVpcSubnet.ID(),
						HostsCount: pulumi.Int(1),
					},
					&DataprocClusterClusterConfigSubclusterSpecArgs{
						Name: pulumi.String("data"),
						Role: pulumi.String("DATANODE"),
						Resources: &DataprocClusterClusterConfigSubclusterSpecResourcesArgs{
							ResourcePresetId: pulumi.String("s2.small"),
							DiskTypeId:       pulumi.String("network-hdd"),
							DiskSize:         pulumi.Int(20),
						},
						SubnetId:   fooVpcSubnet.ID(),
						HostsCount: pulumi.Int(2),
					},
					&DataprocClusterClusterConfigSubclusterSpecArgs{
						Name: pulumi.String("compute"),
						Role: pulumi.String("COMPUTENODE"),
						Resources: &DataprocClusterClusterConfigSubclusterSpecResourcesArgs{
							ResourcePresetId: pulumi.String("s2.small"),
							DiskTypeId:       pulumi.String("network-hdd"),
							DiskSize:         pulumi.Int(20),
						},
						SubnetId:   fooVpcSubnet.ID(),
						HostsCount: pulumi.Int(2),
					},
					&DataprocClusterClusterConfigSubclusterSpecArgs{
						Name: pulumi.String("compute_autoscaling"),
						Role: pulumi.String("COMPUTENODE"),
						Resources: &DataprocClusterClusterConfigSubclusterSpecResourcesArgs{
							ResourcePresetId: pulumi.String("s2.small"),
							DiskTypeId:       pulumi.String("network-hdd"),
							DiskSize:         pulumi.Int(20),
						},
						SubnetId:   fooVpcSubnet.ID(),
						HostsCount: pulumi.Int(2),
						AutoscalingConfig: &DataprocClusterClusterConfigSubclusterSpecAutoscalingConfigArgs{
							MaxHostsCount:         pulumi.Int(10),
							MeasurementDuration:   pulumi.Int(60),
							WarmupDuration:        pulumi.Int(60),
							StabilizationDuration: pulumi.Int(120),
							Preemptible:           pulumi.Bool(false),
							DecommissionTimeout:   pulumi.Int(60),
						},
					},
				},
			},
		}, pulumi.DependsOn([]pulumi.Resource{
			dataprocResourcemanagerFolderIamBinding,
		}))
		if err != nil {
			return err
		}
		return nil
	})
}

import pulumi
import pulumi_yandex as yandex

foo_vpc_network = yandex.VpcNetwork("fooVpcNetwork")
foo_vpc_subnet = yandex.VpcSubnet("fooVpcSubnet",
    zone="ru-central1-b",
    network_id=foo_vpc_network.id,
    v4_cidr_blocks=["10.1.0.0/24"])
dataproc_iam_service_account = yandex.IamServiceAccount("dataprocIamServiceAccount", description="service account to manage Dataproc Cluster")
foo_resourcemanager_folder = yandex.get_resourcemanager_folder(folder_id="some_folder_id")
dataproc_resourcemanager_folder_iam_binding = yandex.ResourcemanagerFolderIamBinding("dataprocResourcemanagerFolderIamBinding",
    folder_id=foo_resourcemanager_folder.id,
    role="mdb.dataproc.agent",
    members=[dataproc_iam_service_account.id.apply(lambda id: f"serviceAccount:{id}")])
# required in order to create bucket
bucket_creator = yandex.ResourcemanagerFolderIamBinding("bucket-creator",
    folder_id=foo_resourcemanager_folder.id,
    role="editor",
    members=[dataproc_iam_service_account.id.apply(lambda id: f"serviceAccount:{id}")])
foo_iam_service_account_static_access_key = yandex.IamServiceAccountStaticAccessKey("fooIamServiceAccountStaticAccessKey", service_account_id=dataproc_iam_service_account.id)
foo_storage_bucket = yandex.StorageBucket("fooStorageBucket",
    bucket="foo",
    access_key=foo_iam_service_account_static_access_key.access_key,
    secret_key=foo_iam_service_account_static_access_key.secret_key,
    opts=pulumi.ResourceOptions(depends_on=[bucket_creator]))
foo_dataproc_cluster = yandex.DataprocCluster("fooDataprocCluster",
    bucket=foo_storage_bucket.bucket,
    description="Dataproc Cluster created by Terraform",
    labels={
        "created_by": "terraform",
    },
    service_account_id=dataproc_iam_service_account.id,
    zone_id="ru-central1-b",
    cluster_config=yandex.DataprocClusterClusterConfigArgs(
        hadoop=yandex.DataprocClusterClusterConfigHadoopArgs(
            services=[
                "HDFS",
                "YARN",
                "SPARK",
                "TEZ",
                "MAPREDUCE",
                "HIVE",
            ],
            properties={
                "yarn:yarn.resourcemanager.am.max-attempts": "5",
            },
            ssh_public_keys=[(lambda path: open(path).read())("~/.ssh/id_rsa.pub")],
        ),
        subcluster_specs=[
            yandex.DataprocClusterClusterConfigSubclusterSpecArgs(
                name="main",
                role="MASTERNODE",
                resources=yandex.DataprocClusterClusterConfigSubclusterSpecResourcesArgs(
                    resource_preset_id="s2.small",
                    disk_type_id="network-hdd",
                    disk_size=20,
                ),
                subnet_id=foo_vpc_subnet.id,
                hosts_count=1,
            ),
            yandex.DataprocClusterClusterConfigSubclusterSpecArgs(
                name="data",
                role="DATANODE",
                resources=yandex.DataprocClusterClusterConfigSubclusterSpecResourcesArgs(
                    resource_preset_id="s2.small",
                    disk_type_id="network-hdd",
                    disk_size=20,
                ),
                subnet_id=foo_vpc_subnet.id,
                hosts_count=2,
            ),
            yandex.DataprocClusterClusterConfigSubclusterSpecArgs(
                name="compute",
                role="COMPUTENODE",
                resources=yandex.DataprocClusterClusterConfigSubclusterSpecResourcesArgs(
                    resource_preset_id="s2.small",
                    disk_type_id="network-hdd",
                    disk_size=20,
                ),
                subnet_id=foo_vpc_subnet.id,
                hosts_count=2,
            ),
            yandex.DataprocClusterClusterConfigSubclusterSpecArgs(
                name="compute_autoscaling",
                role="COMPUTENODE",
                resources=yandex.DataprocClusterClusterConfigSubclusterSpecResourcesArgs(
                    resource_preset_id="s2.small",
                    disk_type_id="network-hdd",
                    disk_size=20,
                ),
                subnet_id=foo_vpc_subnet.id,
                hosts_count=2,
                autoscaling_config=yandex.DataprocClusterClusterConfigSubclusterSpecAutoscalingConfigArgs(
                    max_hosts_count=10,
                    measurement_duration=60,
                    warmup_duration=60,
                    stabilization_duration=120,
                    preemptible=False,
                    decommission_timeout=60,
                ),
            ),
        ],
    ),
    opts=pulumi.ResourceOptions(depends_on=[dataproc_resourcemanager_folder_iam_binding]))

import * as pulumi from "@pulumi/pulumi";
import * as yandex from "@pulumi/yandex";
import * from "fs";

const fooVpcNetwork = new yandex.VpcNetwork("fooVpcNetwork", {});
const fooVpcSubnet = new yandex.VpcSubnet("fooVpcSubnet", {
    zone: "ru-central1-b",
    networkId: fooVpcNetwork.id,
    v4CidrBlocks: ["10.1.0.0/24"],
});
const dataprocIamServiceAccount = new yandex.IamServiceAccount("dataprocIamServiceAccount", {description: "service account to manage Dataproc Cluster"});
const fooResourcemanagerFolder = yandex.getResourcemanagerFolder({
    folderId: "some_folder_id",
});
const dataprocResourcemanagerFolderIamBinding = new yandex.ResourcemanagerFolderIamBinding("dataprocResourcemanagerFolderIamBinding", {
    folderId: fooResourcemanagerFolder.then(fooResourcemanagerFolder => fooResourcemanagerFolder.id),
    role: "mdb.dataproc.agent",
    members: [pulumi.interpolate`serviceAccount:${dataprocIamServiceAccount.id}`],
});
// required in order to create bucket
const bucket_creator = new yandex.ResourcemanagerFolderIamBinding("bucket-creator", {
    folderId: fooResourcemanagerFolder.then(fooResourcemanagerFolder => fooResourcemanagerFolder.id),
    role: "editor",
    members: [pulumi.interpolate`serviceAccount:${dataprocIamServiceAccount.id}`],
});
const fooIamServiceAccountStaticAccessKey = new yandex.IamServiceAccountStaticAccessKey("fooIamServiceAccountStaticAccessKey", {serviceAccountId: dataprocIamServiceAccount.id});
const fooStorageBucket = new yandex.StorageBucket("fooStorageBucket", {
    bucket: "foo",
    accessKey: fooIamServiceAccountStaticAccessKey.accessKey,
    secretKey: fooIamServiceAccountStaticAccessKey.secretKey,
}, {
    dependsOn: [bucket_creator],
});
const fooDataprocCluster = new yandex.DataprocCluster("fooDataprocCluster", {
    bucket: fooStorageBucket.bucket,
    description: "Dataproc Cluster created by Terraform",
    labels: {
        created_by: "terraform",
    },
    serviceAccountId: dataprocIamServiceAccount.id,
    zoneId: "ru-central1-b",
    clusterConfig: {
        hadoop: {
            services: [
                "HDFS",
                "YARN",
                "SPARK",
                "TEZ",
                "MAPREDUCE",
                "HIVE",
            ],
            properties: {
                "yarn:yarn.resourcemanager.am.max-attempts": 5,
            },
            sshPublicKeys: [fs.readFileSync("~/.ssh/id_rsa.pub")],
        },
        subclusterSpecs: [
            {
                name: "main",
                role: "MASTERNODE",
                resources: {
                    resourcePresetId: "s2.small",
                    diskTypeId: "network-hdd",
                    diskSize: 20,
                },
                subnetId: fooVpcSubnet.id,
                hostsCount: 1,
            },
            {
                name: "data",
                role: "DATANODE",
                resources: {
                    resourcePresetId: "s2.small",
                    diskTypeId: "network-hdd",
                    diskSize: 20,
                },
                subnetId: fooVpcSubnet.id,
                hostsCount: 2,
            },
            {
                name: "compute",
                role: "COMPUTENODE",
                resources: {
                    resourcePresetId: "s2.small",
                    diskTypeId: "network-hdd",
                    diskSize: 20,
                },
                subnetId: fooVpcSubnet.id,
                hostsCount: 2,
            },
            {
                name: "compute_autoscaling",
                role: "COMPUTENODE",
                resources: {
                    resourcePresetId: "s2.small",
                    diskTypeId: "network-hdd",
                    diskSize: 20,
                },
                subnetId: fooVpcSubnet.id,
                hostsCount: 2,
                autoscalingConfig: {
                    maxHostsCount: 10,
                    measurementDuration: 60,
                    warmupDuration: 60,
                    stabilizationDuration: 120,
                    preemptible: false,
                    decommissionTimeout: 60,
                },
            },
        ],
    },
}, {
    dependsOn: [dataprocResourcemanagerFolderIamBinding],
});