Merge branch 'start_workflow'

artifacts/example.yaml

@@ -0,0 +1,15 @@
+artefact_url: "https://example.com/artifact.zip"
+type: "zip" # Possible values: zip, tgz
+doi: "XX.XXXX/XXXXXXX.XXXXXXX"
+dockerfiles:
+  - name: "image1:version"
+    location: "path/to/docker/folder"
+    package_managers:
+      - "dpkg" # Possible values: dpkg, rpm, pacman, pip, conda
+    git_packages:
+      - name: "pkg1"
+        location: "path/to/git/repo"
+    misc_packages:
+      - name: "mpkg1"
+        url: "https://example.com/package1.zip"
+        type: "zip" # Possible values: zip, tgz

blacklist.csv

@@ -0,0 +1 @@
+blacklists/blacklist.csv

blacklists/blacklist.csv

@@ -0,0 +1 @@
+sc24_test, IMAGE_NOT_FOUND, 0

flake.lock

@@ -20,16 +20,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1717179513,
+        "lastModified": 1701282334,
-        "narHash": "sha256-vboIEwIQojofItm2xGCdZCzW96U85l9nDW3ifMuAIdM=",
+        "narHash": "sha256-MxCVrXY6v4QmfTwIysjjaX0XUhqBbxTWWB4HXtDYsdk=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "63dacb46bf939521bdc93981b4cbb7ecb58427a0",
+        "rev": "057f9aecfb71c4437d2b27d3323df7f93c010b7e",
         "type": "github"
       },
       "original": {
         "owner": "nixos",
-        "ref": "24.05",
+        "ref": "23.11",
         "repo": "nixpkgs",
         "type": "github"
       }

flake.nix

@@ -2,7 +2,7 @@
   description = "Flake study docker longevity";
 
   inputs = {
-    nixpkgs.url = "github:nixos/nixpkgs/24.05";
+    nixpkgs.url = "github:nixos/nixpkgs/23.11";
     flake-utils.url = "github:numtide/flake-utils";
   };
 
@@ -15,6 +15,8 @@
       devShells = {
         default = pkgs.mkShell {
           packages = with pkgs; [
+            snakemake
+            gawk
             (python3.withPackages (ps: with ps; [
               requests
               pyyaml

workflow/Snakefile

@@ -0,0 +1,41 @@
+include: "utils.smk"
+
+import datetime
+DATE = datetime.datetime.now().strftime("%Y%m%d")
+
+ARTIFACTS_FOLDER = "artifacts"
+BLACKLIST_FOLDER = "blacklists"
+BLACKLIST = "blacklist.csv"
+EXTENSION = "yaml"
+
+ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER, BLACKLIST)
+
+rule all:
+  input:
+    expand("{folder}/{artifact}/{date}.csv",\
+           folder=["logs", "pkgs", "status"],\
+           artifact=ARTIFACTS,\
+           date=DATE),
+    f"{BLACKLIST_FOLDER}/{DATE}.csv"
+
+rule run_ecg:
+  input:
+    "flake.nix",
+    "flake.lock",
+    ecg="ecg.py",
+  output:
+    log    = "logs/{artifact}/{date}.csv",
+    pkg    = "pkgs/{artifact}/{date}.csv",
+    status = "status/{artifact}/{date}.csv",
+  shell:
+    f"python3 {{input.ecg}} --log {{output.log}} --pkg {{output.pkg}} --status {{output.pkg}} {ARTIFACTS_FOLDER}/{{wildcards.artifact}}.{EXTENSION}"
+
+rule update_blacklist:
+  input:
+    BLACKLIST,
+    status=expand("status/{artifact}/{{date}}.csv",\
+                  artifact=ARTIFACTS)
+  output:
+    f"{BLACKLIST_FOLDER}/{{date}}.csv"
+  shell:
+    f"cat {{input}} > {{output}} && ln -s {{output}} {BLACKLIST}"

workflow/utils.smk

@@ -0,0 +1,15 @@
+import csv
+import os
+
+def get_blacklisted_paths(blacklist_csv_path):
+  blacklisted = set()
+  with open(blacklist_csv_path, "r") as csv_file:
+    spamreader = csv.reader(csv_file, delimiter=",")
+    for row in spamreader:
+      blacklisted.add(row[0])
+  return blacklisted
+
+def get_artifacts_to_build(artifacts_folder, blacklist_csv_path):
+  blacklisted = get_blacklisted_paths(blacklist_csv_path)
+  all_artifacts = set([a.split(".")[0] for a in os.listdir(artifacts_folder) if not os.path.isdir(os.path.join(artifacts_folder, a))])
+  return list(all_artifacts.difference(blacklisted))