basic workflow for ecg

blacklist.csv

@@ -1 +0,0 @@
-artifacts/sc24_test.yaml, IMAGE_NOT_FOUND, 0

blacklist.csv

@@ -0,0 +1 @@
+blacklists/blacklist.csv

blacklists/blacklist.csv

@@ -0,0 +1 @@
+sc24_test, IMAGE_NOT_FOUND, 0

workflow/Snakefile

@@ -0,0 +1,41 @@
+include: "utils.smk"
+
+import datetime
+DATE = datetime.datetime.now().strftime("%Y%m%d")
+
+ARTIFACTS_FOLDER = "artifacts"
+BLACKLIST_FOLDER = "blacklists"
+BLACKLIST = "blacklist.csv"
+EXTENSION = "yaml"
+
+ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER, BLACKLIST)
+
+rule all:
+  input:
+    expand("{folder}/{artifact}/{date}.csv",\
+           folder=["logs", "pkgs", "status"],\
+           artifact=ARTIFACTS,\
+           date=DATE),
+    f"{BLACKLIST_FOLDER}/{DATE}.csv"
+
+rule run_ecg:
+  input:
+    "flake.nix",
+    "flake.lock",
+    ecg="ecg.py",
+  output:
+    log    = "logs/{artifact}/{date}.csv",
+    pkg    = "pkgs/{artifact}/{date}.csv",
+    status = "status/{artifact}/{date}.csv",
+  shell:
+    f"python3 {{input.ecg}} --log {{output.log}} --pkg {{output.pkg}} --status {{output.pkg}} {ARTIFACTS_FOLDER}/{{wildcards.artifact}}.{EXTENSION}"
+
+rule update_blacklist:
+  input:
+    BLACKLIST,
+    status=expand("status/{artifact}/{{date}}.csv",\
+                  artifact=ARTIFACTS)
+  output:
+    f"{BLACKLIST_FOLDER}/{{date}}.csv"
+  shell:
+    f"cat {{input}} > {{output}} && ln -s {{output}} {BLACKLIST}"

workflow/utils.smk

@@ -1,9 +1,6 @@
 import csv
 import os
 
-ARTIFACTS_FOLDER = "artifacts"
-BLACKLIST = "blacklist.csv"
-
 def get_blacklisted_paths(blacklist_csv_path):
   blacklisted = set()
   with open(blacklist_csv_path, "r") as csv_file:
@@ -14,15 +11,5 @@ def get_blacklisted_paths(blacklist_csv_path):
 
 def get_artifacts_to_build(artifacts_folder, blacklist_csv_path):
   blacklisted = get_blacklisted_paths(blacklist_csv_path)
-  all_artifacts = set([os.path.join(artifacts_folder, a) for a in os.listdir(artifacts_folder) if not os.path.isdir(os.path.join(artifacts_folder, a))])
+  all_artifacts = set([a.split(".")[0] for a in os.listdir(artifacts_folder) if not os.path.isdir(os.path.join(artifacts_folder, a))])
   return list(all_artifacts.difference(blacklisted))
-
-rule all:
-  input:
-    BLACKLIST,
-  output:
-    "all.csv",
-  params:
-    artifacts = get_artifacts_to_build(ARTIFACTS_FOLDER, BLACKLIST),
-  shell:
-    "echo {params.artifacts} > {output}"