150 lines
4.8 KiB
YAML
150 lines
4.8 KiB
YAML
|
|
apiVersion: argoproj.io/v1alpha1
|
|
kind: Workflow
|
|
metadata:
|
|
generateName: harvesting-
|
|
labels:
|
|
example: 'true'
|
|
workflows.argoproj.io/creator: 0d47b046-a09e-4bed-b10a-ec26783d4fe7
|
|
workflows.argoproj.io/creator-email: pierre.bayle.at.irt-stexupery.com
|
|
workflows.argoproj.io/creator-preferred-username: pbayle
|
|
spec:
|
|
templates:
|
|
- name: busybox
|
|
inputs:
|
|
parameters:
|
|
- name: model
|
|
- name: output-dir
|
|
- name: output-file
|
|
- name: clustername
|
|
outputs:
|
|
parameters:
|
|
- name: outfile
|
|
value: '{{inputs.parameters.output-file}}.tgz'
|
|
artifacts:
|
|
- name: outputs
|
|
path: '{{inputs.parameters.output-dir}}/{{inputs.parameters.output-file}}.tgz'
|
|
s3:
|
|
key: '{{workflow.name}}/{{inputs.parameters.output-file}}.tgz'
|
|
container:
|
|
image: busybox
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
echo "Creating tarball for model: {{inputs.parameters.model}}";
|
|
mkdir -p {{inputs.parameters.output-dir}};
|
|
echo $(ping 8.8.8.8 -c 4) > $(date +%Y-%m-%d__%H-%M-%S)_{{inputs.parameters.output-file}}.txt
|
|
tar -czf {{inputs.parameters.output-dir}}/{{inputs.parameters.output-file}}.tgz *_{{inputs.parameters.output-file}}.txt;
|
|
metadata:
|
|
annotations:
|
|
multicluster.admiralty.io/elect: ""
|
|
multicluster.admiralty.io/clustername: "{{inputs.parameters.clustername}}"
|
|
|
|
- name: weather-container
|
|
inputs:
|
|
parameters:
|
|
- name: output-dir
|
|
- name: output-file
|
|
- name: clustername
|
|
outputs:
|
|
parameters:
|
|
- name: outfile
|
|
value: '{{inputs.parameters.output-file}}.tgz'
|
|
artifacts:
|
|
- name: outputs
|
|
path: '{{inputs.parameters.output-dir}}/{{inputs.parameters.output-file}}.tgz'
|
|
s3:
|
|
insecure: true
|
|
key: '{{workflow.name}}/{{inputs.parameters.output-file}}'
|
|
container:
|
|
name: weather-container
|
|
image: pierrebirt/weather_container:latest
|
|
#imagePullPolicy: IfNotPresent
|
|
env:
|
|
- name: API_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: cnes-secrets
|
|
key: weather-api
|
|
args:
|
|
- '--key'
|
|
- "$(API_KEY)"
|
|
- '--dir'
|
|
- '{{inputs.parameters.output-dir}}'
|
|
- '--file'
|
|
- '{{inputs.parameters.output-file}}'
|
|
metadata:
|
|
annotations:
|
|
multicluster.admiralty.io/elect: ""
|
|
multicluster.admiralty.io/clustername: "{{inputs.parameters.clustername}}"
|
|
|
|
- name: bucket-reader
|
|
inputs:
|
|
parameters:
|
|
- name: bucket-path
|
|
- name: logs-path
|
|
artifacts:
|
|
- name: retrieved-logs
|
|
path: '{{inputs.parameters.logs-path}}'
|
|
s3:
|
|
key: '{{inputs.parameters.bucket-path}}'
|
|
outputs:
|
|
artifacts:
|
|
- name: logs_for_test
|
|
path: /tmp/empty_log_for_test.log
|
|
s3:
|
|
key: '{{workflow.name}}/log_test.log'
|
|
container:
|
|
image: busybox
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
tar -xvf '{{inputs.parameters.logs-path}}'
|
|
ls -la
|
|
cat *.txt
|
|
touch /tmp/empty_log_for_test.log
|
|
|
|
- name: harvesting-test
|
|
inputs: {}
|
|
outputs: {}
|
|
metadata: {}
|
|
dag:
|
|
tasks:
|
|
- name: busybox-dc02
|
|
template: busybox
|
|
arguments:
|
|
parameters:
|
|
- name: model
|
|
value: era-pressure-levels
|
|
- name: output-dir
|
|
value: /app/data/output
|
|
- name: output-file
|
|
value: fake_logs
|
|
- name: clustername
|
|
value: target-dc02
|
|
- name: weather-container-dc03
|
|
template: weather-container
|
|
arguments:
|
|
parameters:
|
|
- name: output-dir
|
|
value: /app/results
|
|
- name: output-file
|
|
value: weather_results_23_01
|
|
- name: clustername
|
|
value: target-dc03
|
|
- name: bucket-reader
|
|
template: bucket-reader
|
|
dependencies: [busybox-dc02,weather-container-dc03]
|
|
arguments:
|
|
parameters:
|
|
- name: bucket-path
|
|
value: '{{workflow.name}}/fake_logs.tgz'
|
|
- name: logs-path
|
|
value: /tmp/logs.tgz
|
|
|
|
entrypoint: harvesting-test
|
|
serviceAccountName: argo-agregateur-workflow-controller
|
|
artifactRepositoryRef: # https://argo-workflows.readthedocs.io/en/latest/fields/#s3artifactrepository
|
|
key: admiralty-s3-artifact-repository # Choose the artifact repository with the public IP/url
|
|
|