Merge branch 'start_workflow'
This commit is contained in:
commit
e06831bd78
15
artifacts/example.yaml
Normal file
15
artifacts/example.yaml
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
artefact_url: "https://example.com/artifact.zip"
|
||||||
|
type: "zip" # Possible values: zip, tgz
|
||||||
|
doi: "XX.XXXX/XXXXXXX.XXXXXXX"
|
||||||
|
dockerfiles:
|
||||||
|
- name: "image1:version"
|
||||||
|
location: "path/to/docker/folder"
|
||||||
|
package_managers:
|
||||||
|
- "dpkg" # Possible values: dpkg, rpm, pacman, pip, conda
|
||||||
|
git_packages:
|
||||||
|
- name: "pkg1"
|
||||||
|
location: "path/to/git/repo"
|
||||||
|
misc_packages:
|
||||||
|
- name: "mpkg1"
|
||||||
|
url: "https://example.com/package1.zip"
|
||||||
|
type: "zip" # Possible values: zip, tgz
|
1
blacklist.csv
Symbolic link
1
blacklist.csv
Symbolic link
@ -0,0 +1 @@
|
|||||||
|
blacklists/blacklist.csv
|
|
1
blacklists/blacklist.csv
Normal file
1
blacklists/blacklist.csv
Normal file
@ -0,0 +1 @@
|
|||||||
|
sc24_test, IMAGE_NOT_FOUND, 0
|
|
@ -20,16 +20,16 @@
|
|||||||
},
|
},
|
||||||
"nixpkgs": {
|
"nixpkgs": {
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1717179513,
|
"lastModified": 1701282334,
|
||||||
"narHash": "sha256-vboIEwIQojofItm2xGCdZCzW96U85l9nDW3ifMuAIdM=",
|
"narHash": "sha256-MxCVrXY6v4QmfTwIysjjaX0XUhqBbxTWWB4HXtDYsdk=",
|
||||||
"owner": "nixos",
|
"owner": "nixos",
|
||||||
"repo": "nixpkgs",
|
"repo": "nixpkgs",
|
||||||
"rev": "63dacb46bf939521bdc93981b4cbb7ecb58427a0",
|
"rev": "057f9aecfb71c4437d2b27d3323df7f93c010b7e",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
"owner": "nixos",
|
"owner": "nixos",
|
||||||
"ref": "24.05",
|
"ref": "23.11",
|
||||||
"repo": "nixpkgs",
|
"repo": "nixpkgs",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
|
@ -2,7 +2,7 @@
|
|||||||
description = "Flake study docker longevity";
|
description = "Flake study docker longevity";
|
||||||
|
|
||||||
inputs = {
|
inputs = {
|
||||||
nixpkgs.url = "github:nixos/nixpkgs/24.05";
|
nixpkgs.url = "github:nixos/nixpkgs/23.11";
|
||||||
flake-utils.url = "github:numtide/flake-utils";
|
flake-utils.url = "github:numtide/flake-utils";
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -15,6 +15,8 @@
|
|||||||
devShells = {
|
devShells = {
|
||||||
default = pkgs.mkShell {
|
default = pkgs.mkShell {
|
||||||
packages = with pkgs; [
|
packages = with pkgs; [
|
||||||
|
snakemake
|
||||||
|
gawk
|
||||||
(python3.withPackages (ps: with ps; [
|
(python3.withPackages (ps: with ps; [
|
||||||
requests
|
requests
|
||||||
pyyaml
|
pyyaml
|
||||||
|
41
workflow/Snakefile
Normal file
41
workflow/Snakefile
Normal file
@ -0,0 +1,41 @@
|
|||||||
|
include: "utils.smk"
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
DATE = datetime.datetime.now().strftime("%Y%m%d")
|
||||||
|
|
||||||
|
ARTIFACTS_FOLDER = "artifacts"
|
||||||
|
BLACKLIST_FOLDER = "blacklists"
|
||||||
|
BLACKLIST = "blacklist.csv"
|
||||||
|
EXTENSION = "yaml"
|
||||||
|
|
||||||
|
ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER, BLACKLIST)
|
||||||
|
|
||||||
|
rule all:
|
||||||
|
input:
|
||||||
|
expand("{folder}/{artifact}/{date}.csv",\
|
||||||
|
folder=["logs", "pkgs", "status"],\
|
||||||
|
artifact=ARTIFACTS,\
|
||||||
|
date=DATE),
|
||||||
|
f"{BLACKLIST_FOLDER}/{DATE}.csv"
|
||||||
|
|
||||||
|
rule run_ecg:
|
||||||
|
input:
|
||||||
|
"flake.nix",
|
||||||
|
"flake.lock",
|
||||||
|
ecg="ecg.py",
|
||||||
|
output:
|
||||||
|
log = "logs/{artifact}/{date}.csv",
|
||||||
|
pkg = "pkgs/{artifact}/{date}.csv",
|
||||||
|
status = "status/{artifact}/{date}.csv",
|
||||||
|
shell:
|
||||||
|
f"python3 {{input.ecg}} --log {{output.log}} --pkg {{output.pkg}} --status {{output.pkg}} {ARTIFACTS_FOLDER}/{{wildcards.artifact}}.{EXTENSION}"
|
||||||
|
|
||||||
|
rule update_blacklist:
|
||||||
|
input:
|
||||||
|
BLACKLIST,
|
||||||
|
status=expand("status/{artifact}/{{date}}.csv",\
|
||||||
|
artifact=ARTIFACTS)
|
||||||
|
output:
|
||||||
|
f"{BLACKLIST_FOLDER}/{{date}}.csv"
|
||||||
|
shell:
|
||||||
|
f"cat {{input}} > {{output}} && ln -s {{output}} {BLACKLIST}"
|
15
workflow/utils.smk
Normal file
15
workflow/utils.smk
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
import csv
|
||||||
|
import os
|
||||||
|
|
||||||
|
def get_blacklisted_paths(blacklist_csv_path):
|
||||||
|
blacklisted = set()
|
||||||
|
with open(blacklist_csv_path, "r") as csv_file:
|
||||||
|
spamreader = csv.reader(csv_file, delimiter=",")
|
||||||
|
for row in spamreader:
|
||||||
|
blacklisted.add(row[0])
|
||||||
|
return blacklisted
|
||||||
|
|
||||||
|
def get_artifacts_to_build(artifacts_folder, blacklist_csv_path):
|
||||||
|
blacklisted = get_blacklisted_paths(blacklist_csv_path)
|
||||||
|
all_artifacts = set([a.split(".")[0] for a in os.listdir(artifacts_folder) if not os.path.isdir(os.path.join(artifacts_folder, a))])
|
||||||
|
return list(all_artifacts.difference(blacklisted))
|
Loading…
Reference in New Issue
Block a user