diff --git a/Workflows/.gitignore b/Workflows/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..895f68583fcf08600643ccf7c79c58d4e7618c4a --- /dev/null +++ b/Workflows/.gitignore @@ -0,0 +1,2 @@ +couler.git +venv diff --git a/Worflow.md b/Workflows/Readme.md similarity index 100% rename from Worflow.md rename to Workflows/Readme.md diff --git a/Workflows/argo-workflows-Testing.md b/Workflows/argo-workflows-Testing.md new file mode 100644 index 0000000000000000000000000000000000000000..254aa2794f76c8442c8f7851cdfef22f5bbafed0 --- /dev/null +++ b/Workflows/argo-workflows-Testing.md @@ -0,0 +1,74 @@ +# First hands-on contact with Argo-Workflows + +## Argo-Workflows server quick start + +Refer to the [install notes](https://argoproj.github.io/argo-workflows/quick-start/#install-argo-workflows) that boils down to + +```bash +minikube --memory 8192 --cpus 4 start +kubectl create ns argo +kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo-workflows/stable/manifests/quick-start-postgres.yaml +kubectl -n argo port-forward deployment/argo-server 2746:2746 +``` + +and open `https://localhost:2746` with your web-browser. + +## Running argo from CLI (yaml workflow) + +For v3.0.8 refer to this [argo CLI install reference](https://github.com/argoproj/argo-workflows/releases/tag/v3.0.8) (look for the "Argo CLI" chapter) that for Mac states + +```bash +curl -sLO https://github.com/argoproj/argo/releases/download/v3.0.8/argo-darwin-amd64.gz +gunzip argo-darwin-amd64.gz +chmod +x argo-darwin-amd64 +mv ./argo-darwin-amd64 /usr/local/bin/argo +argo version +``` + +```bash +argo submit -n argo --watch https://raw.githubusercontent.com/argoproj/argo-workflows/master/examples/hello-world.yaml +``` + +## Running argo from Python-SDK (Couler) examples + +Note: although they seem to exist three Python SDK's for argo, namely +[argo-client-python](https://github.com/argoproj-labs/argo-client-python/), +[argo-python-dsl](https://github.com/argoproj-labs/argo-python-dsl) and +[couler](https://couler-proj.github.io/couler/), only the latter seems to be +still under-development. + +[Couler](https://couler-proj.github.io/couler/), the Argo Workflows Python-SDK, +presents itself as being platform-agnostic enough to provide "a unified +interface for constructing and managing workflows on different workflow +engines, such as Argo Workflows, Tekton Pipelines, and Apache Airflow". + +[Couler documentation](https://couler-proj.github.io/couler/) provides +[some examples](https://couler-proj.github.io/couler/examples/#dag) that +can be run from CLI with Python: + +```bash +$ virtualenv -p python3 venv +$ . venv/bin/activate +(venv)$ pip install git+https://github.com/couler-proj/couler +``` + +```bash +$ git clone https://github.com/couler-proj/couler couler.git +``` + +By default the `couler.git/examples/hello_world.py` runs smoothly but within +the "default" (k8s) namespace that doesn't seem obvious to find under the Web +UI: + +```bash +(venv)$ python couler.git/examples/hello_world.py +``` + +This is why we copied it (`cp couler.git/examples/hello_world.py .`) and +modified to run in the "argo" namespace: + +```bash +(venv)$ python hello_world.py # In CWD not in couler.git +``` + +and observe it running under the UI (in the "argo" namespace). diff --git a/Workflows/dag.py b/Workflows/dag.py new file mode 100644 index 0000000000000000000000000000000000000000..ca6c0003b6c55a01d90d3f8efef4825c54aa280a --- /dev/null +++ b/Workflows/dag.py @@ -0,0 +1,75 @@ +# Copied from https://github.com/couler-proj/couler/examples/hello_world.py +# just to try changing the namespace (from default to argo) +import couler.argo as couler +from couler.argo_submitter import ArgoSubmitter + + +def job_a(message): + couler.run_container( + image="docker/whalesay:latest", + command=["cowsay"], + args=[message], + step_name="A", + ) + + +def job_b(message): + couler.run_container( + image="docker/whalesay:latest", + command=["cowsay"], + args=[message], + step_name="B", + ) + + +def job_c(message): + couler.run_container( + image="docker/whalesay:latest", + command=["cowsay"], + args=[message], + step_name="C", + ) + + +def job_d(message): + couler.run_container( + image="docker/whalesay:latest", + command=["cowsay"], + args=[message], + step_name="D", + ) + + +# A +# / \ +# B C +# / +# D +def linear(): + couler.set_dependencies(lambda: job_a(message="A"), dependencies=None) + couler.set_dependencies(lambda: job_b(message="B"), dependencies=["A"]) + couler.set_dependencies(lambda: job_c(message="C"), dependencies=["A"]) + couler.set_dependencies(lambda: job_d(message="D"), dependencies=["B"]) + + +# A +# / \ +# B C +# \ / +# D +def diamond(): + couler.dag( + [ + [lambda: job_a(message="A")], + [lambda: job_a(message="A"), lambda: job_b(message="B")], # A -> B + [lambda: job_a(message="A"), lambda: job_c(message="C")], # A -> C + [lambda: job_b(message="B"), lambda: job_d(message="D")], # B -> D + [lambda: job_c(message="C"), lambda: job_d(message="D")], # C -> D + ] + ) + + +# linear() +diamond() +submitter = ArgoSubmitter(namespace='argo') +couler.run(submitter=submitter) diff --git a/Workflows/hello_world.py b/Workflows/hello_world.py new file mode 100644 index 0000000000000000000000000000000000000000..a197c294c8a5f8ca59f6afdab14e86f34aafc7a4 --- /dev/null +++ b/Workflows/hello_world.py @@ -0,0 +1,11 @@ +# Copied from https://github.com/couler-proj/couler/examples/hello_world.py +# just to try changing the namespace (from default to argo) +import couler.argo as couler +from couler.argo_submitter import ArgoSubmitter + +couler.run_container( + image="docker/whalesay", command=["cowsay"], args=["hello world"] +) + +submitter = ArgoSubmitter(namespace='argo') +result = couler.run(submitter=submitter)