Skip to content

Webhdfs Input Output Artifacts

This example showcases how to use WebHDFS.

This example used to reside in the examples directory, but was moved here to avoid the extra overwrite parameter specified in the outputs field upstream, which is not an allowed parameter. Example: - https://github.com/argoproj/argo-workflows/blob/master/examples/webhdfs-input-output-artifacts.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
from hera.workflows import (
    Container,
    HTTPArtifact,
    Workflow,
    models as m,
)

with Workflow(
    generate_name="input-output-artifact-webhdfs-",
    entrypoint="input-output-artifact-webhdfs-example",
) as w:
    Container(
        name="input-output-artifact-webhdfs-example",
        image="debian:latest",
        command=["sh", "-c"],
        args=["cat /my-artifact"],
        inputs=[
            HTTPArtifact(
                name="my-art",
                path="/my-artifact",
                url="https://mywebhdfsprovider.com/webhdfs/v1/file.txt?op=OPEN",
                auth=m.HTTPAuth(
                    oauth2=m.OAuth2Auth(
                        client_id_secret=m.SecretKeySelector(name="oauth-sec", key="clientID"),
                        client_secret_secret=m.SecretKeySelector(name="oauth-sec", key="clientSecret"),
                        token_url_secret=m.SecretKeySelector(
                            name="oauth-sec",
                            key="tokenURL",
                        ),
                        scopes=["some", "scopes"],
                        endpoint_params=[
                            m.OAuth2EndpointParam(key="customkey", value="customvalue"),
                        ],
                    ),
                ),
                headers=[
                    m.Header(name="CustomHeader", value="CustomValue"),
                ],
            )
        ],
        outputs=[
            HTTPArtifact(
                name="my-art2",
                path="/my-artifact",
                url="https://mywebhdfsprovider.com/webhdfs/v1/file.txt?op=CREATE&overwrite=true",
                auth=m.HTTPAuth(
                    client_cert=m.ClientCertAuth(
                        client_cert_secret=m.SecretKeySelector(
                            name="cert-sec",
                            key="certificate.pem",
                        ),
                        client_key_secret=m.SecretKeySelector(
                            name="cert-sec",
                            key="key.pem",
                        ),
                    )
                ),
                headers=[m.Header(name="CustomHeader", value="CustomValue")],
            )
        ],
    )

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
  generateName: input-output-artifact-webhdfs-
spec:
  entrypoint: input-output-artifact-webhdfs-example
  templates:
  - container:
      args:
      - cat /my-artifact
      command:
      - sh
      - -c
      image: debian:latest
    inputs:
      artifacts:
      - http:
          auth:
            oauth2:
              clientIDSecret:
                key: clientID
                name: oauth-sec
              clientSecretSecret:
                key: clientSecret
                name: oauth-sec
              endpointParams:
              - key: customkey
                value: customvalue
              scopes:
              - some
              - scopes
              tokenURLSecret:
                key: tokenURL
                name: oauth-sec
          headers:
          - name: CustomHeader
            value: CustomValue
          url: https://mywebhdfsprovider.com/webhdfs/v1/file.txt?op=OPEN
        name: my-art
        path: /my-artifact
    name: input-output-artifact-webhdfs-example
    outputs:
      artifacts:
      - http:
          auth:
            clientCert:
              clientCertSecret:
                key: certificate.pem
                name: cert-sec
              clientKeySecret:
                key: key.pem
                name: cert-sec
          headers:
          - name: CustomHeader
            value: CustomValue
          url: https://mywebhdfsprovider.com/webhdfs/v1/file.txt?op=CREATE&overwrite=true
        name: my-art2
        path: /my-artifact

Comments