From 9289b10b8471833ac2fa128c125db0f8c4e85288 Mon Sep 17 00:00:00 2001
From: Quentin Guilloteau <Quentin.Guilloteau@unibas.ch>
Date: Thu, 29 Aug 2024 13:10:24 +0200
Subject: [PATCH] start splitting the workflow into several pieces

---
 workflow/measure.smk | 63 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)
 create mode 100644 workflow/measure.smk

diff --git a/workflow/measure.smk b/workflow/measure.smk
new file mode 100644
index 0000000..5fe7d2b
--- /dev/null
+++ b/workflow/measure.smk
@@ -0,0 +1,63 @@
+configfile: "config/config.yaml"
+
+include: "utils.smk"
+
+import os
+
+ARTIFACTS_FOLDER_NICKEL = config["folder_artifacts_nickel"]
+ARTIFACTS_FOLDER_JSON   = config["folder_artifacts_json"]
+SYSTEM = config["system"]
+PREFIX = config["prefix"]
+
+rule main:
+  input:
+    lambda w: expand(f"{PREFIX}/{{{{conference}}}}/build_status/{{artifact}}/{{{{date}}}}.csv",\
+           artifact=get_artifacts_to_build(ARTIFACTS_FOLDER_NICKEL + "/" + w['conference']))
+  output:
+    "{conference}_{date}.ok"
+  shell:
+    "echo {input} > {output}"
+
+rule check_artifact:
+  input:
+    "flake.nix",
+    "flake.lock",
+    contract="workflow/nickel/artifact_contract.ncl",
+    artifact=f"{ARTIFACTS_FOLDER_NICKEL}/{{conference}}/{{artifact}}.ncl"
+  output:
+    f"{ARTIFACTS_FOLDER_JSON}/{{conference}}/{{artifact}}.json"
+  shell:
+    """
+    nix develop .#nickel --command nickel export --format json --output {output} <<< 'let {{Artifact, ..}} = import "{input.contract}" in ((import "{input.artifact}") | Artifact)'
+    """
+
+# ECG:
+
+rule run_ecg:
+  input:
+    "flake.nix",
+    "flake.lock",
+    ecg="ecg/app/ecg.py",
+    execo_wrapper="workflow/scripts/submission_g5k.py",
+    oar_wrapper="workflow/scripts/ecg_oar_wrapper.oar.bash",
+    artifact=f"{ARTIFACTS_FOLDER_JSON}/{{conference}}/{{artifact}}.json"
+  output:
+    log           = f"{PREFIX}/{{conference}}/logs/{{artifact}}/{{date}}.txt",
+    pkg           = f"{PREFIX}/{{conference}}/pkgs/{{artifact}}/{{date}}.csv",
+    build_status  = f"{PREFIX}/{{conference}}/build_status/{{artifact}}/{{date}}.csv",
+    artifact_hash = f"{PREFIX}/{{conference}}/artifact_hash/{{artifact}}/{{date}}.csv",
+  shell:
+    (f"python3 {{input.execo_wrapper}} --path {os.getcwd()} \
+                                       --script {{input.oar_wrapper}} \
+                                       --site {config['site']} \
+                                       --cluster {config['cluster']} \
+                                       --max-duration {config['max_duration']} \
+                                       --checkpoint {config['checkpoint']} \
+                                     {'--besteffort' if config['besteffort'] else ''} \
+                                       --sleep_time {config['sleep_time']} \
+                                       --build_status_file {{output.build_status}} \
+                                       --artifact {{wildcards.artifact}} -- '" if SYSTEM == "g5k" else "") + \
+    """
+    nix shell .#ecg --command ecg -p {output.pkg} -b {output.build_status} -a {output.artifact_hash} {input.artifact} &> {output.log} || echo "{input.artifact}, `date +%s.%N`, script_crash" > {output.build_status}
+    """ + \
+    ("'" if SYSTEM == "g5k" else "")