Docs Home Pulumi IaC Pulumi ESC Pulumi Cloud Packages Tutorials
  1. Packages
  2. Google Cloud (GCP) Classic
  3. API Docs
  4. dataflow
  5. Job
Google Cloud Classic v8.9.3 published on Monday, Nov 18, 2024 by Pulumi
pulumi/pulumi-gcp

gcp.dataflow.Job

Explore with Pulumi AI

gcp logo
Google Cloud Classic v8.9.3 published on Monday, Nov 18, 2024 by Pulumi
pulumi/pulumi-gcp

    Creates a job on Dataflow, which is an implementation of Apache Beam running on Google Compute Engine. For more information see the official documentation for Beam and Dataflow.

    Example Usage

    import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";

const bigDataJob = new gcp.dataflow.Job("big_data_job", {
    name: "dataflow-job",
    templateGcsPath: "gs://my-bucket/templates/template_file",
    tempGcsLocation: "gs://my-bucket/tmp_dir",
    parameters: {
        foo: "bar",
        baz: "qux",
    },
});

    import pulumi
import pulumi_gcp as gcp

big_data_job = gcp.dataflow.Job("big_data_job",
    name="dataflow-job",
    template_gcs_path="gs://my-bucket/templates/template_file",
    temp_gcs_location="gs://my-bucket/tmp_dir",
    parameters={
        "foo": "bar",
        "baz": "qux",
    })

    package main

import (
	"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/dataflow"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		_, err := dataflow.NewJob(ctx, "big_data_job", &dataflow.JobArgs{
			Name:            pulumi.String("dataflow-job"),
			TemplateGcsPath: pulumi.String("gs://my-bucket/templates/template_file"),
			TempGcsLocation: pulumi.String("gs://my-bucket/tmp_dir"),
			Parameters: pulumi.StringMap{
				"foo": pulumi.String("bar"),
				"baz": pulumi.String("qux"),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

    using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;

return await Deployment.RunAsync(() => 
{
    var bigDataJob = new Gcp.Dataflow.Job("big_data_job", new()
    {
        Name = "dataflow-job",
        TemplateGcsPath = "gs://my-bucket/templates/template_file",
        TempGcsLocation = "gs://my-bucket/tmp_dir",
        Parameters = 
        {
            { "foo", "bar" },
            { "baz", "qux" },
        },
    });

});

    package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.dataflow.Job;
import com.pulumi.gcp.dataflow.JobArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var bigDataJob = new Job("bigDataJob", JobArgs.builder()
            .name("dataflow-job")
            .templateGcsPath("gs://my-bucket/templates/template_file")
            .tempGcsLocation("gs://my-bucket/tmp_dir")
            .parameters(Map.ofEntries(
                Map.entry("foo", "bar"),
                Map.entry("baz", "qux")
            ))
            .build());

    }
}

    resources:
  bigDataJob:
    type: gcp:dataflow:Job
    name: big_data_job
    properties:
      name: dataflow-job
      templateGcsPath: gs://my-bucket/templates/template_file
      tempGcsLocation: gs://my-bucket/tmp_dir
      parameters:
        foo: bar
        baz: qux

    Streaming Job

    import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";

const topic = new gcp.pubsub.Topic("topic", {name: "dataflow-job1"});
const bucket1 = new gcp.storage.Bucket("bucket1", {
    name: "tf-test-bucket1",
    location: "US",
    forceDestroy: true,
});
const bucket2 = new gcp.storage.Bucket("bucket2", {
    name: "tf-test-bucket2",
    location: "US",
    forceDestroy: true,
});
const pubsubStream = new gcp.dataflow.Job("pubsub_stream", {
    name: "tf-test-dataflow-job1",
    templateGcsPath: "gs://my-bucket/templates/template_file",
    tempGcsLocation: "gs://my-bucket/tmp_dir",
    enableStreamingEngine: true,
    parameters: {
        inputFilePattern: pulumi.interpolate`${bucket1.url}/*.json`,
        outputTopic: topic.id,
    },
    transformNameMapping: {
        name: "test_job",
        env: "test",
    },
    onDelete: "cancel",
});

    import pulumi
import pulumi_gcp as gcp

topic = gcp.pubsub.Topic("topic", name="dataflow-job1")
bucket1 = gcp.storage.Bucket("bucket1",
    name="tf-test-bucket1",
    location="US",
    force_destroy=True)
bucket2 = gcp.storage.Bucket("bucket2",
    name="tf-test-bucket2",
    location="US",
    force_destroy=True)
pubsub_stream = gcp.dataflow.Job("pubsub_stream",
    name="tf-test-dataflow-job1",
    template_gcs_path="gs://my-bucket/templates/template_file",
    temp_gcs_location="gs://my-bucket/tmp_dir",
    enable_streaming_engine=True,
    parameters={
        "inputFilePattern": bucket1.url.apply(lambda url: f"{url}/*.json"),
        "outputTopic": topic.id,
    },
    transform_name_mapping={
        "name": "test_job",
        "env": "test",
    },
    on_delete="cancel")

    package main

import (
	"fmt"

	"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/dataflow"
	"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/pubsub"
	"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/storage"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		topic, err := pubsub.NewTopic(ctx, "topic", &pubsub.TopicArgs{
			Name: pulumi.String("dataflow-job1"),
		})
		if err != nil {
			return err
		}
		bucket1, err := storage.NewBucket(ctx, "bucket1", &storage.BucketArgs{
			Name:         pulumi.String("tf-test-bucket1"),
			Location:     pulumi.String("US"),
			ForceDestroy: pulumi.Bool(true),
		})
		if err != nil {
			return err
		}
		_, err = storage.NewBucket(ctx, "bucket2", &storage.BucketArgs{
			Name:         pulumi.String("tf-test-bucket2"),
			Location:     pulumi.String("US"),
			ForceDestroy: pulumi.Bool(true),
		})
		if err != nil {
			return err
		}
		_, err = dataflow.NewJob(ctx, "pubsub_stream", &dataflow.JobArgs{
			Name:                  pulumi.String("tf-test-dataflow-job1"),
			TemplateGcsPath:       pulumi.String("gs://my-bucket/templates/template_file"),
			TempGcsLocation:       pulumi.String("gs://my-bucket/tmp_dir"),
			EnableStreamingEngine: pulumi.Bool(true),
			Parameters: pulumi.StringMap{
				"inputFilePattern": bucket1.Url.ApplyT(func(url string) (string, error) {
					return fmt.Sprintf("%v/*.json", url), nil
				}).(pulumi.StringOutput),
				"outputTopic": topic.ID(),
			},
			TransformNameMapping: pulumi.StringMap{
				"name": pulumi.String("test_job"),
				"env":  pulumi.String("test"),
			},
			OnDelete: pulumi.String("cancel"),
		})
		if err != nil {
			return err
		}
		return nil
	})
}

    using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;

return await Deployment.RunAsync(() => 
{
    var topic = new Gcp.PubSub.Topic("topic", new()
    {
        Name = "dataflow-job1",
    });

    var bucket1 = new Gcp.Storage.Bucket("bucket1", new()
    {
        Name = "tf-test-bucket1",
        Location = "US",
        ForceDestroy = true,
    });

    var bucket2 = new Gcp.Storage.Bucket("bucket2", new()
    {
        Name = "tf-test-bucket2",
        Location = "US",
        ForceDestroy = true,
    });

    var pubsubStream = new Gcp.Dataflow.Job("pubsub_stream", new()
    {
        Name = "tf-test-dataflow-job1",
        TemplateGcsPath = "gs://my-bucket/templates/template_file",
        TempGcsLocation = "gs://my-bucket/tmp_dir",
        EnableStreamingEngine = true,
        Parameters = 
        {
            { "inputFilePattern", bucket1.Url.Apply(url => $"{url}/*.json") },
            { "outputTopic", topic.Id },
        },
        TransformNameMapping = 
        {
            { "name", "test_job" },
            { "env", "test" },
        },
        OnDelete = "cancel",
    });

});

    package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.pubsub.Topic;
import com.pulumi.gcp.pubsub.TopicArgs;
import com.pulumi.gcp.storage.Bucket;
import com.pulumi.gcp.storage.BucketArgs;
import com.pulumi.gcp.dataflow.Job;
import com.pulumi.gcp.dataflow.JobArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        var topic = new Topic("topic", TopicArgs.builder()
            .name("dataflow-job1")
            .build());

        var bucket1 = new Bucket("bucket1", BucketArgs.builder()
            .name("tf-test-bucket1")
            .location("US")
            .forceDestroy(true)
            .build());

        var bucket2 = new Bucket("bucket2", BucketArgs.builder()
            .name("tf-test-bucket2")
            .location("US")
            .forceDestroy(true)
            .build());

        var pubsubStream = new Job("pubsubStream", JobArgs.builder()
            .name("tf-test-dataflow-job1")
            .templateGcsPath("gs://my-bucket/templates/template_file")
            .tempGcsLocation("gs://my-bucket/tmp_dir")
            .enableStreamingEngine(true)
            .parameters(Map.ofEntries(
                Map.entry("inputFilePattern", bucket1.url().applyValue(url -> String.format("%s/*.json", url))),
                Map.entry("outputTopic", topic.id())
            ))
            .transformNameMapping(Map.ofEntries(
                Map.entry("name", "test_job"),
                Map.entry("env", "test")
            ))
            .onDelete("cancel")
            .build());

    }
}

    resources:
  topic:
    type: gcp:pubsub:Topic
    properties:
      name: dataflow-job1
  bucket1:
    type: gcp:storage:Bucket
    properties:
      name: tf-test-bucket1
      location: US
      forceDestroy: true
  bucket2:
    type: gcp:storage:Bucket
    properties:
      name: tf-test-bucket2
      location: US
      forceDestroy: true
  pubsubStream:
    type: gcp:dataflow:Job
    name: pubsub_stream
    properties:
      name: tf-test-dataflow-job1
      templateGcsPath: gs://my-bucket/templates/template_file
      tempGcsLocation: gs://my-bucket/tmp_dir
      enableStreamingEngine: true
      parameters:
        inputFilePattern: ${bucket1.url}/*.json
        outputTopic: ${topic.id}
      transformNameMapping:
        name: test_job
        env: test
      onDelete: cancel

    Note on “destroy” / “apply”

    There are many types of Dataflow jobs. Some Dataflow jobs run constantly, getting new data from (e.g.) a GCS bucket, and outputting data continuously. Some jobs process a set amount of data then terminate. All jobs can fail while running due to programming errors or other issues. In this way, Dataflow jobs are different from most other Google resources.

    The Dataflow resource is considered ’existing’ while it is in a nonterminal state. If it reaches a terminal state (e.g. ‘FAILED’, ‘COMPLETE’, ‘CANCELLED’), it will be recreated on the next ‘apply’. This is as expected for jobs which run continuously, but may surprise users who use this resource for other kinds of Dataflow jobs.

    A Dataflow job which is ‘destroyed’ may be “cancelled” or “drained”. If “cancelled”, the job terminates - any data written remains where it is, but no new data will be processed. If “drained”, no new data will enter the pipeline, but any data currently in the pipeline will finish being processed. The default is “drain”. When on_delete is set to "drain" in the configuration, you may experience a long wait for your pulumi destroy to complete.

    You can potentially short-circuit the wait by setting skip_wait_on_job_termination to true, but beware that unless you take active steps to ensure that the job name parameter changes between instances, the name will conflict and the launch of the new job will fail. One way to do this is with a random_id resource, for example:

    import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";
import * as random from "@pulumi/random";

const config = new pulumi.Config();
const bigDataJobSubscriptionId = config.get("bigDataJobSubscriptionId") || "projects/myproject/subscriptions/messages";
const bigDataJobNameSuffix = new random.RandomId("big_data_job_name_suffix", {
    byteLength: 4,
    keepers: {
        region: region,
        subscription_id: bigDataJobSubscriptionId,
    },
});
const bigDataJob = new gcp.dataflow.FlexTemplateJob("big_data_job", {
    name: pulumi.interpolate`dataflow-flextemplates-job-${bigDataJobNameSuffix.dec}`,
    region: region,
    containerSpecGcsPath: "gs://my-bucket/templates/template.json",
    skipWaitOnJobTermination: true,
    parameters: {
        inputSubscription: bigDataJobSubscriptionId,
    },
});

    import pulumi
import pulumi_gcp as gcp
import pulumi_random as random

config = pulumi.Config()
big_data_job_subscription_id = config.get("bigDataJobSubscriptionId")
if big_data_job_subscription_id is None:
    big_data_job_subscription_id = "projects/myproject/subscriptions/messages"
big_data_job_name_suffix = random.RandomId("big_data_job_name_suffix",
    byte_length=4,
    keepers={
        "region": region,
        "subscription_id": big_data_job_subscription_id,
    })
big_data_job = gcp.dataflow.FlexTemplateJob("big_data_job",
    name=big_data_job_name_suffix.dec.apply(lambda dec: f"dataflow-flextemplates-job-{dec}"),
    region=region,
    container_spec_gcs_path="gs://my-bucket/templates/template.json",
    skip_wait_on_job_termination=True,
    parameters={
        "inputSubscription": big_data_job_subscription_id,
    })

    package main

import (
	"fmt"

	"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/dataflow"
	"github.com/pulumi/pulumi-random/sdk/v4/go/random"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		cfg := config.New(ctx, "")
		bigDataJobSubscriptionId := "projects/myproject/subscriptions/messages"
		if param := cfg.Get("bigDataJobSubscriptionId"); param != "" {
			bigDataJobSubscriptionId = param
		}
		bigDataJobNameSuffix, err := random.NewRandomId(ctx, "big_data_job_name_suffix", &random.RandomIdArgs{
			ByteLength: pulumi.Int(4),
			Keepers: pulumi.StringMap{
				"region":          pulumi.Any(region),
				"subscription_id": pulumi.String(bigDataJobSubscriptionId),
			},
		})
		if err != nil {
			return err
		}
		_, err = dataflow.NewFlexTemplateJob(ctx, "big_data_job", &dataflow.FlexTemplateJobArgs{
			Name: bigDataJobNameSuffix.Dec.ApplyT(func(dec string) (string, error) {
				return fmt.Sprintf("dataflow-flextemplates-job-%v", dec), nil
			}).(pulumi.StringOutput),
			Region:                   pulumi.Any(region),
			ContainerSpecGcsPath:     pulumi.String("gs://my-bucket/templates/template.json"),
			SkipWaitOnJobTermination: pulumi.Bool(true),
			Parameters: pulumi.StringMap{
				"inputSubscription": pulumi.String(bigDataJobSubscriptionId),
			},
		})
		if err != nil {
			return err
		}
		return nil
	})
}

    using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;
using Random = Pulumi.Random;

return await Deployment.RunAsync(() => 
{
    var config = new Config();
    var bigDataJobSubscriptionId = config.Get("bigDataJobSubscriptionId") ?? "projects/myproject/subscriptions/messages";
    var bigDataJobNameSuffix = new Random.RandomId("big_data_job_name_suffix", new()
    {
        ByteLength = 4,
        Keepers = 
        {
            { "region", region },
            { "subscription_id", bigDataJobSubscriptionId },
        },
    });

    var bigDataJob = new Gcp.Dataflow.FlexTemplateJob("big_data_job", new()
    {
        Name = bigDataJobNameSuffix.Dec.Apply(dec => $"dataflow-flextemplates-job-{dec}"),
        Region = region,
        ContainerSpecGcsPath = "gs://my-bucket/templates/template.json",
        SkipWaitOnJobTermination = true,
        Parameters = 
        {
            { "inputSubscription", bigDataJobSubscriptionId },
        },
    });

});

    package generated_program;

import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.random.RandomId;
import com.pulumi.random.RandomIdArgs;
import com.pulumi.gcp.dataflow.FlexTemplateJob;
import com.pulumi.gcp.dataflow.FlexTemplateJobArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;

public class App {
    public static void main(String[] args) {
        Pulumi.run(App::stack);
    }

    public static void stack(Context ctx) {
        final var config = ctx.config();
        final var bigDataJobSubscriptionId = config.get("bigDataJobSubscriptionId").orElse("projects/myproject/subscriptions/messages");
        var bigDataJobNameSuffix = new RandomId("bigDataJobNameSuffix", RandomIdArgs.builder()
            .byteLength(4)
            .keepers(Map.ofEntries(
                Map.entry("region", region),
                Map.entry("subscription_id", bigDataJobSubscriptionId)
            ))
            .build());

        var bigDataJob = new FlexTemplateJob("bigDataJob", FlexTemplateJobArgs.builder()
            .name(bigDataJobNameSuffix.dec().applyValue(dec -> String.format("dataflow-flextemplates-job-%s", dec)))
            .region(region)
            .containerSpecGcsPath("gs://my-bucket/templates/template.json")
            .skipWaitOnJobTermination(true)
            .parameters(Map.of("inputSubscription", bigDataJobSubscriptionId))
            .build());

    }
}

    configuration:
  bigDataJobSubscriptionId:
    type: string
    default: projects/myproject/subscriptions/messages
resources:
  bigDataJobNameSuffix:
    type: random:RandomId
    name: big_data_job_name_suffix
    properties:
      byteLength: 4
      keepers:
        region: ${region}
        subscription_id: ${bigDataJobSubscriptionId}
  bigDataJob:
    type: gcp:dataflow:FlexTemplateJob
    name: big_data_job
    properties:
      name: dataflow-flextemplates-job-${bigDataJobNameSuffix.dec}
      region: ${region}
      containerSpecGcsPath: gs://my-bucket/templates/template.json
      skipWaitOnJobTermination: true
      parameters:
        inputSubscription: ${bigDataJobSubscriptionId}

    Create Job Resource

    Resources are created with functions called constructors. To learn more about declaring and configuring resources, see Resources.

    Constructor syntax

    new Job(name: string, args: JobArgs, opts?: CustomResourceOptions);
    @overload
def Job(resource_name: str,
        args: JobArgs,
        opts: Optional[ResourceOptions] = None)

@overload
def Job(resource_name: str,
        opts: Optional[ResourceOptions] = None,
        temp_gcs_location: Optional[str] = None,
        template_gcs_path: Optional[str] = None,
        parameters: Optional[Mapping[str, str]] = None,
        project: Optional[str] = None,
        labels: Optional[Mapping[str, str]] = None,
        machine_type: Optional[str] = None,
        max_workers: Optional[int] = None,
        name: Optional[str] = None,
        network: Optional[str] = None,
        on_delete: Optional[str] = None,
        additional_experiments: Optional[Sequence[str]] = None,
        kms_key_name: Optional[str] = None,
        region: Optional[str] = None,
        service_account_email: Optional[str] = None,
        skip_wait_on_job_termination: Optional[bool] = None,
        subnetwork: Optional[str] = None,
        ip_configuration: Optional[str] = None,
        enable_streaming_engine: Optional[bool] = None,
        transform_name_mapping: Optional[Mapping[str, str]] = None,
        zone: Optional[str] = None)
    func NewJob(ctx *Context, name string, args JobArgs, opts ...ResourceOption) (*Job, error)
    public Job(string name, JobArgs args, CustomResourceOptions? opts = null)
    public Job(String name, JobArgs args)
public Job(String name, JobArgs args, CustomResourceOptions options)

    type: gcp:dataflow:Job
properties: # The arguments to resource properties.
options: # Bag of options to control resource's behavior.

    Parameters

    name string
    The unique name of the resource.
    args JobArgs
    The arguments to resource properties.
    opts CustomResourceOptions
    Bag of options to control resource's behavior.
    resource_name str
    The unique name of the resource.
    args JobArgs
    The arguments to resource properties.
    opts ResourceOptions
    Bag of options to control resource's behavior.
    ctx Context
    Context object for the current deployment.
    name string
    The unique name of the resource.
    args JobArgs
    The arguments to resource properties.
    opts ResourceOption
    Bag of options to control resource's behavior.
    name string
    The unique name of the resource.
    args JobArgs
    The arguments to resource properties.
    opts CustomResourceOptions
    Bag of options to control resource's behavior.
    name String
    The unique name of the resource.
    args JobArgs
    The arguments to resource properties.
    options CustomResourceOptions
    Bag of options to control resource's behavior.

    Constructor example

    The following reference example uses placeholder values for all input properties.

    var examplejobResourceResourceFromDataflowjob = new Gcp.Dataflow.Job("examplejobResourceResourceFromDataflowjob", new()
{
    TempGcsLocation = "string",
    TemplateGcsPath = "string",
    Parameters = 
    {
        { "string", "string" },
    },
    Project = "string",
    Labels = 
    {
        { "string", "string" },
    },
    MachineType = "string",
    MaxWorkers = 0,
    Name = "string",
    Network = "string",
    OnDelete = "string",
    AdditionalExperiments = new[]
    {
        "string",
    },
    KmsKeyName = "string",
    Region = "string",
    ServiceAccountEmail = "string",
    SkipWaitOnJobTermination = false,
    Subnetwork = "string",
    IpConfiguration = "string",
    EnableStreamingEngine = false,
    TransformNameMapping = 
    {
        { "string", "string" },
    },
    Zone = "string",
});

    example, err := dataflow.NewJob(ctx, "examplejobResourceResourceFromDataflowjob", &dataflow.JobArgs{
	TempGcsLocation: pulumi.String("string"),
	TemplateGcsPath: pulumi.String("string"),
	Parameters: pulumi.StringMap{
		"string": pulumi.String("string"),
	},
	Project: pulumi.String("string"),
	Labels: pulumi.StringMap{
		"string": pulumi.String("string"),
	},
	MachineType: pulumi.String("string"),
	MaxWorkers:  pulumi.Int(0),
	Name:        pulumi.String("string"),
	Network:     pulumi.String("string"),
	OnDelete:    pulumi.String("string"),
	AdditionalExperiments: pulumi.StringArray{
		pulumi.String("string"),
	},
	KmsKeyName:               pulumi.String("string"),
	Region:                   pulumi.String("string"),
	ServiceAccountEmail:      pulumi.String("string"),
	SkipWaitOnJobTermination: pulumi.Bool(false),
	Subnetwork:               pulumi.String("string"),
	IpConfiguration:          pulumi.String("string"),
	EnableStreamingEngine:    pulumi.Bool(false),
	TransformNameMapping: pulumi.StringMap{
		"string": pulumi.String("string"),
	},
	Zone: pulumi.String("string"),
})

    var examplejobResourceResourceFromDataflowjob = new Job("examplejobResourceResourceFromDataflowjob", JobArgs.builder()
    .tempGcsLocation("string")
    .templateGcsPath("string")
    .parameters(Map.of("string", "string"))
    .project("string")
    .labels(Map.of("string", "string"))
    .machineType("string")
    .maxWorkers(0)
    .name("string")
    .network("string")
    .onDelete("string")
    .additionalExperiments("string")
    .kmsKeyName("string")
    .region("string")
    .serviceAccountEmail("string")
    .skipWaitOnJobTermination(false)
    .subnetwork("string")
    .ipConfiguration("string")
    .enableStreamingEngine(false)
    .transformNameMapping(Map.of("string", "string"))
    .zone("string")
    .build());

    examplejob_resource_resource_from_dataflowjob = gcp.dataflow.Job("examplejobResourceResourceFromDataflowjob",
    temp_gcs_location="string",
    template_gcs_path="string",
    parameters={
        "string": "string",
    },
    project="string",
    labels={
        "string": "string",
    },
    machine_type="string",
    max_workers=0,
    name="string",
    network="string",
    on_delete="string",
    additional_experiments=["string"],
    kms_key_name="string",
    region="string",
    service_account_email="string",
    skip_wait_on_job_termination=False,
    subnetwork="string",
    ip_configuration="string",
    enable_streaming_engine=False,
    transform_name_mapping={
        "string": "string",
    },
    zone="string")

    const examplejobResourceResourceFromDataflowjob = new gcp.dataflow.Job("examplejobResourceResourceFromDataflowjob", {
    tempGcsLocation: "string",
    templateGcsPath: "string",
    parameters: {
        string: "string",
    },
    project: "string",
    labels: {
        string: "string",
    },
    machineType: "string",
    maxWorkers: 0,
    name: "string",
    network: "string",
    onDelete: "string",
    additionalExperiments: ["string"],
    kmsKeyName: "string",
    region: "string",
    serviceAccountEmail: "string",
    skipWaitOnJobTermination: false,
    subnetwork: "string",
    ipConfiguration: "string",
    enableStreamingEngine: false,
    transformNameMapping: {
        string: "string",
    },
    zone: "string",
});

    type: gcp:dataflow:Job
properties:
    additionalExperiments:
        - string
    enableStreamingEngine: false
    ipConfiguration: string
    kmsKeyName: string
    labels:
        string: string
    machineType: string
    maxWorkers: 0
    name: string
    network: string
    onDelete: string
    parameters:
        string: string
    project: string
    region: string
    serviceAccountEmail: string
    skipWaitOnJobTermination: false
    subnetwork: string
    tempGcsLocation: string
    templateGcsPath: string
    transformNameMapping:
        string: string
    zone: string

    Job Resource Properties

    To learn more about resource properties and how to use them, see Inputs and Outputs in the Architecture and Concepts docs.

    Inputs

    In Python, inputs that are objects can be passed either as argument classes or as dictionary literals.

    The Job resource accepts the following input properties:

    TempGcsLocation string
    A writeable location on GCS for the Dataflow job to dump its temporary data.

    TemplateGcsPath string
    The GCS path to the Dataflow job template.
    AdditionalExperiments List<string>
    List of experiments that should be used by the job. An example value is ["enable_stackdriver_agent_metrics"].
    EnableStreamingEngine bool
    Enable/disable the use of Streaming Engine for the job. Note that Streaming Engine is enabled by default for pipelines developed against the Beam SDK for Python v2.21.0 or later when using Python 3.
    IpConfiguration string
    The configuration for VM IPs. Options are "WORKER_IP_PUBLIC" or "WORKER_IP_PRIVATE".
    KmsKeyName string
    The name for the Cloud KMS key for the job. Key format is: projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY
    Labels Dictionary<string, string>
    User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field effective_labels for all of the labels present on the resource.
    MachineType string
    The machine type to use for the job.
    MaxWorkers int
    The number of workers permitted to work on the job. More workers may improve processing speed at additional cost.
    Name string
    A unique name for the resource, required by Dataflow.
    Network string
    The network to which VMs will be assigned. If it is not provided, "default" will be used.
    OnDelete string
    One of "drain" or "cancel". Specifies behavior of deletion during pulumi destroy. See above note.
    Parameters Dictionary<string, string>
    Template specific Key/Value pairs to be forwarded to the pipeline's options; keys are case-sensitive based on the language on which the pipeline is coded, mostly Java. Note: do not configure Dataflow options here in parameters.
    Project string
    The project in which the resource belongs. If it is not provided, the provider project is used.
    Region string
    The region in which the created job should run.
    ServiceAccountEmail string
    The Service Account email used to create the job. This should be just an email e.g. myserviceaccount@myproject.iam.gserviceaccount.com. Do not include any serviceAccount: or other prefix.
    SkipWaitOnJobTermination bool
    If set to true, Pulumi will treat DRAINING and CANCELLING as terminal states when deleting the resource, and will remove the resource from Pulumi state and move on. See above note.
    Subnetwork string
    The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in a Shared VPC network, you must use the complete URL. For example "googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME"
    TransformNameMapping Dictionary<string, string>
    Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job. This field is not used outside of update.
    Zone string
    The zone in which the created job should run. If it is not provided, the provider zone is used.
    TempGcsLocation string
    A writeable location on GCS for the Dataflow job to dump its temporary data.

    TemplateGcsPath string
    The GCS path to the Dataflow job template.
    AdditionalExperiments []string
    List of experiments that should be used by the job. An example value is ["enable_stackdriver_agent_metrics"].
    EnableStreamingEngine bool
    Enable/disable the use of Streaming Engine for the job. Note that Streaming Engine is enabled by default for pipelines developed against the Beam SDK for Python v2.21.0 or later when using Python 3.
    IpConfiguration string
    The configuration for VM IPs. Options are "WORKER_IP_PUBLIC" or "WORKER_IP_PRIVATE".
    KmsKeyName string
    The name for the Cloud KMS key for the job. Key format is: projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY
    Labels map[string]string
    User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field effective_labels for all of the labels present on the resource.
    MachineType string
    The machine type to use for the job.
    MaxWorkers int
    The number of workers permitted to work on the job. More workers may improve processing speed at additional cost.
    Name string
    A unique name for the resource, required by Dataflow.
    Network string
    The network to which VMs will be assigned. If it is not provided, "default" will be used.
    OnDelete string
    One of "drain" or "cancel". Specifies behavior of deletion during pulumi destroy. See above note.
    Parameters map[string]string
    Template specific Key/Value pairs to be forwarded to the pipeline's options; keys are case-sensitive based on the language on which the pipeline is coded, mostly Java. Note: do not configure Dataflow options here in parameters.
    Project string
    The project in which the resource belongs. If it is not provided, the provider project is used.
    Region string
    The region in which the created job should run.
    ServiceAccountEmail string
    The Service Account email used to create the job. This should be just an email e.g. myserviceaccount@myproject.iam.gserviceaccount.com. Do not include any serviceAccount: or other prefix.
    SkipWaitOnJobTermination bool
    If set to true, Pulumi will treat DRAINING and CANCELLING as terminal states when deleting the resource, and will remove the resource from Pulumi state and move on. See above note.
    Subnetwork string
    The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in a Shared VPC network, you must use the complete URL. For example "googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME"
    TransformNameMapping map[string]string
    Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job. This field is not used outside of update.
    Zone string
    The zone in which the created job should run. If it is not provided, the provider zone is used.
    tempGcsLocation String
    A writeable location on GCS for the Dataflow job to dump its temporary data.

    templateGcsPath String
    The GCS path to the Dataflow job template.
    additionalExperiments List<String>
    List of experiments that should be used by the job. An example value is ["enable_stackdriver_agent_metrics"].
    enableStreamingEngine Boolean
    Enable/disable the use of Streaming Engine for the job. Note that Streaming Engine is enabled by default for pipelines developed against the Beam SDK for Python v2.21.0 or later when using Python 3.
    ipConfiguration String
    The configuration for VM IPs. Options are "WORKER_IP_PUBLIC" or "WORKER_IP_PRIVATE".
    kmsKeyName String
    The name for the Cloud KMS key for the job. Key format is: projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY
    labels Map<String,String>
    User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field effective_labels for all of the labels present on the resource.
    machineType String
    The machine type to use for the job.
    maxWorkers Integer
    The number of workers permitted to work on the job. More workers may improve processing speed at additional cost.
    name String
    A unique name for the resource, required by Dataflow.
    network String
    The network to which VMs will be assigned. If it is not provided, "default" will be used.
    onDelete String
    One of "drain" or "cancel". Specifies behavior of deletion during pulumi destroy. See above note.
    parameters Map<String,String>
    Template specific Key/Value pairs to be forwarded to the pipeline's options; keys are case-sensitive based on the language on which the pipeline is coded, mostly Java. Note: do not configure Dataflow options here in parameters.
    project String
    The project in which the resource belongs. If it is not provided, the provider project is used.
    region String
    The region in which the created job should run.
    serviceAccountEmail String
    The Service Account email used to create the job. This should be just an email e.g. myserviceaccount@myproject.iam.gserviceaccount.com. Do not include any serviceAccount: or other prefix.
    skipWaitOnJobTermination Boolean
    If set to true, Pulumi will treat DRAINING and CANCELLING as terminal states when deleting the resource, and will remove the resource from Pulumi state and move on. See above note.
    subnetwork String
    The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in a Shared VPC network, you must use the complete URL. For example "googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME"
    transformNameMapping Map<String,String>
    Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job. This field is not used outside of update.
    zone String
    The zone in which the created job should run. If it is not provided, the provider zone is used.
    tempGcsLocation string
    A writeable location on GCS for the Dataflow job to dump its temporary data.

    templateGcsPath string
    The GCS path to the Dataflow job template.
    additionalExperiments string[]
    List of experiments that should be used by the job. An example value is ["enable_stackdriver_agent_metrics"].
    enableStreamingEngine boolean
    Enable/disable the use of Streaming Engine for the job. Note that Streaming Engine is enabled by default for pipelines developed against the Beam SDK for Python v2.21.0 or later when using Python 3.
    ipConfiguration string
    The configuration for VM IPs. Options are "WORKER_IP_PUBLIC" or "WORKER_IP_PRIVATE".
    kmsKeyName string
    The name for the Cloud KMS key for the job. Key format is: projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY
    labels {[key: string]: string}
    User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field effective_labels for all of the labels present on the resource.
    machineType string
    The machine type to use for the job.
    maxWorkers number
    The number of workers permitted to work on the job. More workers may improve processing speed at additional cost.
    name string
    A unique name for the resource, required by Dataflow.
    network string
    The network to which VMs will be assigned. If it is not provided, "default" will be used.
    onDelete string
    One of "drain" or "cancel". Specifies behavior of deletion during pulumi destroy. See above note.
    parameters {[key: string]: string}
    Template specific Key/Value pairs to be forwarded to the pipeline's options; keys are case-sensitive based on the language on which the pipeline is coded, mostly Java. Note: do not configure Dataflow options here in parameters.
    project string
    The project in which the resource belongs. If it is not provided, the provider project is used.
    region string
    The region in which the created job should run.
    serviceAccountEmail string
    The Service Account email used to create the job. This should be just an email e.g. myserviceaccount@myproject.iam.gserviceaccount.com. Do not include any serviceAccount: or other prefix.
    skipWaitOnJobTermination boolean
    If set to true, Pulumi will treat DRAINING and CANCELLING as terminal states when deleting the resource, and will remove the resource from Pulumi state and move on. See above note.
    subnetwork string
    The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in a Shared VPC network, you must use the complete URL. For example "googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME"
    transformNameMapping {[key: string]: string}
    Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job. This field is not used outside of update.
    zone string
    The zone in which the created job should run. If it is not provided, the provider zone is used.
    temp_gcs_location str
    A writeable location on GCS for the Dataflow job to dump its temporary data.

    template_gcs_path str
    The GCS path to the Dataflow job template.
    additional_experiments Sequence[str]
    List of experiments that should be used by the job. An example value is ["enable_stackdriver_agent_metrics"].
    enable_streaming_engine bool
    Enable/disable the use of Streaming Engine for the job. Note that Streaming Engine is enabled by default for pipelines developed against the Beam SDK for Python v2.21.0 or later when using Python 3.
    ip_configuration str
    The configuration for VM IPs. Options are "WORKER_IP_PUBLIC" or "WORKER_IP_PRIVATE".
    kms_key_name str
    The name for the Cloud KMS key for the job. Key format is: projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY
    labels Mapping[str, str]
    User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field effective_labels for all of the labels present on the resource.
    machine_type str
    The machine type to use for the job.
    max_workers int
    The number of workers permitted to work on the job. More workers may improve processing speed at additional cost.
    name str
    A unique name for the resource, required by Dataflow.
    network str
    The network to which VMs will be assigned. If it is not provided, "default" will be used.
    on_delete str
    One of "drain" or "cancel". Specifies behavior of deletion during pulumi destroy. See above note.
    parameters Mapping[str, str]
    Template specific Key/Value pairs to be forwarded to the pipeline's options; keys are case-sensitive based on the language on which the pipeline is coded, mostly Java. Note: do not configure Dataflow options here in parameters.
    project str
    The project in which the resource belongs. If it is not provided, the provider project is used.
    region str
    The region in which the created job should run.
    service_account_email str
    The Service Account email used to create the job. This should be just an email e.g. myserviceaccount@myproject.iam.gserviceaccount.com. Do not include any serviceAccount: or other prefix.
    skip_wait_on_job_termination bool
    If set to true, Pulumi will treat DRAINING and CANCELLING as terminal states when deleting the resource, and will remove the resource from Pulumi state and move on. See above note.
    subnetwork str
    The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in a Shared VPC network, you must use the complete URL. For example "googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME"
    transform_name_mapping Mapping[str, str]
    Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job. This field is not used outside of update.
    zone str
    The zone in which the created job should run. If it is not provided, the provider zone is used.
    tempGcsLocation String
    A writeable location on GCS for the Dataflow job to dump its temporary data.

    templateGcsPath String
    The GCS path to the Dataflow job template.
    additionalExperiments List<String>
    List of experiments that should be used by the job. An example value is ["enable_stackdriver_agent_metrics"].
    enableStreamingEngine Boolean
    Enable/disable the use of Streaming Engine for the job. Note that Streaming Engine is enabled by default for pipelines developed against the Beam SDK for Python v2.21.0 or later when using Python 3.
    ipConfiguration String
    The configuration for VM IPs. Options are "WORKER_IP_PUBLIC" or "WORKER_IP_PRIVATE".
    kmsKeyName String
    The name for the Cloud KMS key for the job. Key format is: projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY
    labels Map<String>
    User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field effective_labels for all of the labels present on the resource.
    machineType String
    The machine type to use for the job.
    maxWorkers Number
    The number of workers permitted to work on the job. More workers may improve processing speed at additional cost.
    name String
    A unique name for the resource, required by Dataflow.
    network String
    The network to which VMs will be assigned. If it is not provided, "default" will be used.
    onDelete String
    One of "drain" or "cancel". Specifies behavior of deletion during pulumi destroy. See above note.
    parameters Map<String>
    Template specific Key/Value pairs to be forwarded to the pipeline's options; keys are case-sensitive based on the language on which the pipeline is coded, mostly Java. Note: do not configure Dataflow options here in parameters.
    project String
    The project in which the resource belongs. If it is not provided, the provider project is used.
    region String
    The region in which the created job should run.
    serviceAccountEmail String
    The Service Account email used to create the job. This should be just an email e.g. myserviceaccount@myproject.iam.gserviceaccount.com. Do not include any serviceAccount: or other prefix.
    skipWaitOnJobTermination Boolean
    If set to true, Pulumi will treat DRAINING and CANCELLING as terminal states when deleting the resource, and will remove the resource from Pulumi state and move on. See above note.
    subnetwork String
    The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in a Shared VPC network, you must use the complete URL. For example "googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME"
    transformNameMapping Map<String>
    Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job. This field is not used outside of update.
    zone String
    The zone in which the created job should run. If it is not provided, the provider zone is used.

    Outputs

    All input properties are implicitly available as output properties. Additionally, the Job resource produces the following output properties:

    EffectiveLabels Dictionary<string, string>
    All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
    Id string
    The provider-assigned unique ID for this managed resource.
    JobId string
    The unique ID of this job.
    PulumiLabels Dictionary<string, string>
    The combination of labels configured directly on the resource and default labels configured on the provider.
    State string
    The current state of the resource, selected from the JobState enum
    Type string
    The type of this job, selected from the JobType enum
    EffectiveLabels map[string]string
    All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
    Id string
    The provider-assigned unique ID for this managed resource.
    JobId string
    The unique ID of this job.
    PulumiLabels map[string]string
    The combination of labels configured directly on the resource and default labels configured on the provider.
    State string
    The current state of the resource, selected from the JobState enum
    Type string
    The type of this job, selected from the JobType enum
    effectiveLabels Map<String,String>
    All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
    id String
    The provider-assigned unique ID for this managed resource.
    jobId String
    The unique ID of this job.
    pulumiLabels Map<String,String>
    The combination of labels configured directly on the resource and default labels configured on the provider.
    state String
    The current state of the resource, selected from the JobState enum
    type String
    The type of this job, selected from the JobType enum
    effectiveLabels {[key: string]: string}
    All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
    id string
    The provider-assigned unique ID for this managed resource.
    jobId string
    The unique ID of this job.
    pulumiLabels {[key: string]: string}
    The combination of labels configured directly on the resource and default labels configured on the provider.
    state string
    The current state of the resource, selected from the JobState enum
    type string
    The type of this job, selected from the JobType enum
    effective_labels Mapping[str, str]
    All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
    id str
    The provider-assigned unique ID for this managed resource.
    job_id str
    The unique ID of this job.
    pulumi_labels Mapping[str, str]
    The combination of labels configured directly on the resource and default labels configured on the provider.
    state str
    The current state of the resource, selected from the JobState enum
    type str
    The type of this job, selected from the JobType enum
    effectiveLabels Map<String>
    All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
    id String
    The provider-assigned unique ID for this managed resource.
    jobId String
    The unique ID of this job.
    pulumiLabels Map<String>
    The combination of labels configured directly on the resource and default labels configured on the provider.
    state String
    The current state of the resource, selected from the JobState enum
    type String
    The type of this job, selected from the JobType enum

    Look up Existing Job Resource

    Get an existing Job resource’s state with the given name, ID, and optional extra properties used to qualify the lookup.

    public static get(name: string, id: Input<ID>, state?: JobState, opts?: CustomResourceOptions): Job
    @staticmethod
def get(resource_name: str,
        id: str,
        opts: Optional[ResourceOptions] = None,
        additional_experiments: Optional[Sequence[str]] = None,
        effective_labels: Optional[Mapping[str, str]] = None,
        enable_streaming_engine: Optional[bool] = None,
        ip_configuration: Optional[str] = None,
        job_id: Optional[str] = None,
        kms_key_name: Optional[str] = None,
        labels: Optional[Mapping[str, str]] = None,
        machine_type: Optional[str] = None,
        max_workers: Optional[int] = None,
        name: Optional[str] = None,
        network: Optional[str] = None,
        on_delete: Optional[str] = None,
        parameters: Optional[Mapping[str, str]] = None,
        project: Optional[str] = None,
        pulumi_labels: Optional[Mapping[str, str]] = None,
        region: Optional[str] = None,
        service_account_email: Optional[str] = None,
        skip_wait_on_job_termination: Optional[bool] = None,
        state: Optional[str] = None,
        subnetwork: Optional[str] = None,
        temp_gcs_location: Optional[str] = None,
        template_gcs_path: Optional[str] = None,
        transform_name_mapping: Optional[Mapping[str, str]] = None,
        type: Optional[str] = None,
        zone: Optional[str] = None) -> Job
    func GetJob(ctx *Context, name string, id IDInput, state *JobState, opts ...ResourceOption) (*Job, error)
    public static Job Get(string name, Input<string> id, JobState? state, CustomResourceOptions? opts = null)
    public static Job get(String name, Output<String> id, JobState state, CustomResourceOptions options)
    Resource lookup is not supported in YAML
    name
    The unique name of the resulting resource.
    id
    The unique provider ID of the resource to lookup.
    state
    Any extra arguments used during the lookup.
    opts
    A bag of options that control this resource's behavior.
    resource_name
    The unique name of the resulting resource.
    id
    The unique provider ID of the resource to lookup.
    name
    The unique name of the resulting resource.
    id
    The unique provider ID of the resource to lookup.
    state
    Any extra arguments used during the lookup.
    opts
    A bag of options that control this resource's behavior.
    name
    The unique name of the resulting resource.
    id
    The unique provider ID of the resource to lookup.
    state
    Any extra arguments used during the lookup.
    opts
    A bag of options that control this resource's behavior.
    name
    The unique name of the resulting resource.
    id
    The unique provider ID of the resource to lookup.
    state
    Any extra arguments used during the lookup.
    opts
    A bag of options that control this resource's behavior.
    The following state arguments are supported:
    AdditionalExperiments List<string>
    List of experiments that should be used by the job. An example value is ["enable_stackdriver_agent_metrics"].
    EffectiveLabels Dictionary<string, string>
    All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
    EnableStreamingEngine bool
    Enable/disable the use of Streaming Engine for the job. Note that Streaming Engine is enabled by default for pipelines developed against the Beam SDK for Python v2.21.0 or later when using Python 3.
    IpConfiguration string
    The configuration for VM IPs. Options are "WORKER_IP_PUBLIC" or "WORKER_IP_PRIVATE".
    JobId string
    The unique ID of this job.
    KmsKeyName string
    The name for the Cloud KMS key for the job. Key format is: projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY
    Labels Dictionary<string, string>
    User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field effective_labels for all of the labels present on the resource.
    MachineType string
    The machine type to use for the job.
    MaxWorkers int
    The number of workers permitted to work on the job. More workers may improve processing speed at additional cost.
    Name string
    A unique name for the resource, required by Dataflow.
    Network string
    The network to which VMs will be assigned. If it is not provided, "default" will be used.
    OnDelete string
    One of "drain" or "cancel". Specifies behavior of deletion during pulumi destroy. See above note.
    Parameters Dictionary<string, string>
    Template specific Key/Value pairs to be forwarded to the pipeline's options; keys are case-sensitive based on the language on which the pipeline is coded, mostly Java. Note: do not configure Dataflow options here in parameters.
    Project string
    The project in which the resource belongs. If it is not provided, the provider project is used.
    PulumiLabels Dictionary<string, string>
    The combination of labels configured directly on the resource and default labels configured on the provider.
    Region string
    The region in which the created job should run.
    ServiceAccountEmail string
    The Service Account email used to create the job. This should be just an email e.g. myserviceaccount@myproject.iam.gserviceaccount.com. Do not include any serviceAccount: or other prefix.
    SkipWaitOnJobTermination bool
    If set to true, Pulumi will treat DRAINING and CANCELLING as terminal states when deleting the resource, and will remove the resource from Pulumi state and move on. See above note.
    State string
    The current state of the resource, selected from the JobState enum
    Subnetwork string
    The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in a Shared VPC network, you must use the complete URL. For example "googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME"
    TempGcsLocation string
    A writeable location on GCS for the Dataflow job to dump its temporary data.

    TemplateGcsPath string
    The GCS path to the Dataflow job template.
    TransformNameMapping Dictionary<string, string>
    Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job. This field is not used outside of update.
    Type string
    The type of this job, selected from the JobType enum
    Zone string
    The zone in which the created job should run. If it is not provided, the provider zone is used.
    AdditionalExperiments []string
    List of experiments that should be used by the job. An example value is ["enable_stackdriver_agent_metrics"].
    EffectiveLabels map[string]string
    All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
    EnableStreamingEngine bool
    Enable/disable the use of Streaming Engine for the job. Note that Streaming Engine is enabled by default for pipelines developed against the Beam SDK for Python v2.21.0 or later when using Python 3.
    IpConfiguration string
    The configuration for VM IPs. Options are "WORKER_IP_PUBLIC" or "WORKER_IP_PRIVATE".
    JobId string
    The unique ID of this job.
    KmsKeyName string
    The name for the Cloud KMS key for the job. Key format is: projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY
    Labels map[string]string
    User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field effective_labels for all of the labels present on the resource.
    MachineType string
    The machine type to use for the job.
    MaxWorkers int
    The number of workers permitted to work on the job. More workers may improve processing speed at additional cost.
    Name string
    A unique name for the resource, required by Dataflow.
    Network string
    The network to which VMs will be assigned. If it is not provided, "default" will be used.
    OnDelete string
    One of "drain" or "cancel". Specifies behavior of deletion during pulumi destroy. See above note.
    Parameters map[string]string
    Template specific Key/Value pairs to be forwarded to the pipeline's options; keys are case-sensitive based on the language on which the pipeline is coded, mostly Java. Note: do not configure Dataflow options here in parameters.
    Project string
    The project in which the resource belongs. If it is not provided, the provider project is used.
    PulumiLabels map[string]string
    The combination of labels configured directly on the resource and default labels configured on the provider.
    Region string
    The region in which the created job should run.
    ServiceAccountEmail string
    The Service Account email used to create the job. This should be just an email e.g. myserviceaccount@myproject.iam.gserviceaccount.com. Do not include any serviceAccount: or other prefix.
    SkipWaitOnJobTermination bool
    If set to true, Pulumi will treat DRAINING and CANCELLING as terminal states when deleting the resource, and will remove the resource from Pulumi state and move on. See above note.
    State string
    The current state of the resource, selected from the JobState enum
    Subnetwork string
    The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in a Shared VPC network, you must use the complete URL. For example "googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME"
    TempGcsLocation string
    A writeable location on GCS for the Dataflow job to dump its temporary data.

    TemplateGcsPath string
    The GCS path to the Dataflow job template.
    TransformNameMapping map[string]string
    Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job. This field is not used outside of update.
    Type string
    The type of this job, selected from the JobType enum
    Zone string
    The zone in which the created job should run. If it is not provided, the provider zone is used.
    additionalExperiments List<String>
    List of experiments that should be used by the job. An example value is ["enable_stackdriver_agent_metrics"].
    effectiveLabels Map<String,String>
    All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
    enableStreamingEngine Boolean
    Enable/disable the use of Streaming Engine for the job. Note that Streaming Engine is enabled by default for pipelines developed against the Beam SDK for Python v2.21.0 or later when using Python 3.
    ipConfiguration String
    The configuration for VM IPs. Options are "WORKER_IP_PUBLIC" or "WORKER_IP_PRIVATE".
    jobId String
    The unique ID of this job.
    kmsKeyName String
    The name for the Cloud KMS key for the job. Key format is: projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY
    labels Map<String,String>
    User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field effective_labels for all of the labels present on the resource.
    machineType String
    The machine type to use for the job.
    maxWorkers Integer
    The number of workers permitted to work on the job. More workers may improve processing speed at additional cost.
    name String
    A unique name for the resource, required by Dataflow.
    network String
    The network to which VMs will be assigned. If it is not provided, "default" will be used.
    onDelete String
    One of "drain" or "cancel". Specifies behavior of deletion during pulumi destroy. See above note.
    parameters Map<String,String>
    Template specific Key/Value pairs to be forwarded to the pipeline's options; keys are case-sensitive based on the language on which the pipeline is coded, mostly Java. Note: do not configure Dataflow options here in parameters.
    project String
    The project in which the resource belongs. If it is not provided, the provider project is used.
    pulumiLabels Map<String,String>
    The combination of labels configured directly on the resource and default labels configured on the provider.
    region String
    The region in which the created job should run.
    serviceAccountEmail String
    The Service Account email used to create the job. This should be just an email e.g. myserviceaccount@myproject.iam.gserviceaccount.com. Do not include any serviceAccount: or other prefix.
    skipWaitOnJobTermination Boolean
    If set to true, Pulumi will treat DRAINING and CANCELLING as terminal states when deleting the resource, and will remove the resource from Pulumi state and move on. See above note.
    state String
    The current state of the resource, selected from the JobState enum
    subnetwork String
    The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in a Shared VPC network, you must use the complete URL. For example "googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME"
    tempGcsLocation String
    A writeable location on GCS for the Dataflow job to dump its temporary data.

    templateGcsPath String
    The GCS path to the Dataflow job template.
    transformNameMapping Map<String,String>
    Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job. This field is not used outside of update.
    type String
    The type of this job, selected from the JobType enum
    zone String
    The zone in which the created job should run. If it is not provided, the provider zone is used.
    additionalExperiments string[]
    List of experiments that should be used by the job. An example value is ["enable_stackdriver_agent_metrics"].
    effectiveLabels {[key: string]: string}
    All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
    enableStreamingEngine boolean
    Enable/disable the use of Streaming Engine for the job. Note that Streaming Engine is enabled by default for pipelines developed against the Beam SDK for Python v2.21.0 or later when using Python 3.
    ipConfiguration string
    The configuration for VM IPs. Options are "WORKER_IP_PUBLIC" or "WORKER_IP_PRIVATE".
    jobId string
    The unique ID of this job.
    kmsKeyName string
    The name for the Cloud KMS key for the job. Key format is: projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY
    labels {[key: string]: string}
    User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field effective_labels for all of the labels present on the resource.
    machineType string
    The machine type to use for the job.
    maxWorkers number
    The number of workers permitted to work on the job. More workers may improve processing speed at additional cost.
    name string
    A unique name for the resource, required by Dataflow.
    network string
    The network to which VMs will be assigned. If it is not provided, "default" will be used.
    onDelete string
    One of "drain" or "cancel". Specifies behavior of deletion during pulumi destroy. See above note.
    parameters {[key: string]: string}
    Template specific Key/Value pairs to be forwarded to the pipeline's options; keys are case-sensitive based on the language on which the pipeline is coded, mostly Java. Note: do not configure Dataflow options here in parameters.
    project string
    The project in which the resource belongs. If it is not provided, the provider project is used.
    pulumiLabels {[key: string]: string}
    The combination of labels configured directly on the resource and default labels configured on the provider.
    region string
    The region in which the created job should run.
    serviceAccountEmail string
    The Service Account email used to create the job. This should be just an email e.g. myserviceaccount@myproject.iam.gserviceaccount.com. Do not include any serviceAccount: or other prefix.
    skipWaitOnJobTermination boolean
    If set to true, Pulumi will treat DRAINING and CANCELLING as terminal states when deleting the resource, and will remove the resource from Pulumi state and move on. See above note.
    state string
    The current state of the resource, selected from the JobState enum
    subnetwork string
    The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in a Shared VPC network, you must use the complete URL. For example "googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME"
    tempGcsLocation string
    A writeable location on GCS for the Dataflow job to dump its temporary data.

    templateGcsPath string
    The GCS path to the Dataflow job template.
    transformNameMapping {[key: string]: string}
    Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job. This field is not used outside of update.
    type string
    The type of this job, selected from the JobType enum
    zone string
    The zone in which the created job should run. If it is not provided, the provider zone is used.
    additional_experiments Sequence[str]
    List of experiments that should be used by the job. An example value is ["enable_stackdriver_agent_metrics"].
    effective_labels Mapping[str, str]
    All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
    enable_streaming_engine bool
    Enable/disable the use of Streaming Engine for the job. Note that Streaming Engine is enabled by default for pipelines developed against the Beam SDK for Python v2.21.0 or later when using Python 3.
    ip_configuration str
    The configuration for VM IPs. Options are "WORKER_IP_PUBLIC" or "WORKER_IP_PRIVATE".
    job_id str
    The unique ID of this job.
    kms_key_name str
    The name for the Cloud KMS key for the job. Key format is: projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY
    labels Mapping[str, str]
    User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field effective_labels for all of the labels present on the resource.
    machine_type str
    The machine type to use for the job.
    max_workers int
    The number of workers permitted to work on the job. More workers may improve processing speed at additional cost.
    name str
    A unique name for the resource, required by Dataflow.
    network str
    The network to which VMs will be assigned. If it is not provided, "default" will be used.
    on_delete str
    One of "drain" or "cancel". Specifies behavior of deletion during pulumi destroy. See above note.
    parameters Mapping[str, str]
    Template specific Key/Value pairs to be forwarded to the pipeline's options; keys are case-sensitive based on the language on which the pipeline is coded, mostly Java. Note: do not configure Dataflow options here in parameters.
    project str
    The project in which the resource belongs. If it is not provided, the provider project is used.
    pulumi_labels Mapping[str, str]
    The combination of labels configured directly on the resource and default labels configured on the provider.
    region str
    The region in which the created job should run.
    service_account_email str
    The Service Account email used to create the job. This should be just an email e.g. myserviceaccount@myproject.iam.gserviceaccount.com. Do not include any serviceAccount: or other prefix.
    skip_wait_on_job_termination bool
    If set to true, Pulumi will treat DRAINING and CANCELLING as terminal states when deleting the resource, and will remove the resource from Pulumi state and move on. See above note.
    state str
    The current state of the resource, selected from the JobState enum
    subnetwork str
    The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in a Shared VPC network, you must use the complete URL. For example "googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME"
    temp_gcs_location str
    A writeable location on GCS for the Dataflow job to dump its temporary data.

    template_gcs_path str
    The GCS path to the Dataflow job template.
    transform_name_mapping Mapping[str, str]
    Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job. This field is not used outside of update.
    type str
    The type of this job, selected from the JobType enum
    zone str
    The zone in which the created job should run. If it is not provided, the provider zone is used.
    additionalExperiments List<String>
    List of experiments that should be used by the job. An example value is ["enable_stackdriver_agent_metrics"].
    effectiveLabels Map<String>
    All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services.
    enableStreamingEngine Boolean
    Enable/disable the use of Streaming Engine for the job. Note that Streaming Engine is enabled by default for pipelines developed against the Beam SDK for Python v2.21.0 or later when using Python 3.
    ipConfiguration String
    The configuration for VM IPs. Options are "WORKER_IP_PUBLIC" or "WORKER_IP_PRIVATE".
    jobId String
    The unique ID of this job.
    kmsKeyName String
    The name for the Cloud KMS key for the job. Key format is: projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY
    labels Map<String>
    User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. Note: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field effective_labels for all of the labels present on the resource.
    machineType String
    The machine type to use for the job.
    maxWorkers Number
    The number of workers permitted to work on the job. More workers may improve processing speed at additional cost.
    name String
    A unique name for the resource, required by Dataflow.
    network String
    The network to which VMs will be assigned. If it is not provided, "default" will be used.
    onDelete String
    One of "drain" or "cancel". Specifies behavior of deletion during pulumi destroy. See above note.
    parameters Map<String>
    Template specific Key/Value pairs to be forwarded to the pipeline's options; keys are case-sensitive based on the language on which the pipeline is coded, mostly Java. Note: do not configure Dataflow options here in parameters.
    project String
    The project in which the resource belongs. If it is not provided, the provider project is used.
    pulumiLabels Map<String>
    The combination of labels configured directly on the resource and default labels configured on the provider.
    region String
    The region in which the created job should run.
    serviceAccountEmail String
    The Service Account email used to create the job. This should be just an email e.g. myserviceaccount@myproject.iam.gserviceaccount.com. Do not include any serviceAccount: or other prefix.
    skipWaitOnJobTermination Boolean
    If set to true, Pulumi will treat DRAINING and CANCELLING as terminal states when deleting the resource, and will remove the resource from Pulumi state and move on. See above note.
    state String
    The current state of the resource, selected from the JobState enum
    subnetwork String
    The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in a Shared VPC network, you must use the complete URL. For example "googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/subnetworks/SUBNET_NAME"
    tempGcsLocation String
    A writeable location on GCS for the Dataflow job to dump its temporary data.

    templateGcsPath String
    The GCS path to the Dataflow job template.
    transformNameMapping Map<String>
    Only applicable when updating a pipeline. Map of transform name prefixes of the job to be replaced with the corresponding name prefixes of the new job. This field is not used outside of update.
    type String
    The type of this job, selected from the JobType enum
    zone String
    The zone in which the created job should run. If it is not provided, the provider zone is used.

    Import

    Dataflow jobs can be imported using the job id e.g.

    • {{id}}

    When using the pulumi import command, dataflow jobs can be imported using one of the formats above. For example:

    $ pulumi import gcp:dataflow/job:Job default {{id}}

    To learn more about importing existing cloud resources, see Importing resources.

    Package Details

    Repository
    Google Cloud (GCP) Classic pulumi/pulumi-gcp
    License
    Apache-2.0
    Notes
    This Pulumi package is based on the google-beta Terraform Provider.
    gcp logo
    Google Cloud Classic v8.9.3 published on Monday, Nov 18, 2024 by Pulumi
    pulumi/pulumi-gcp