2024-07-19 16:33:27 +02:00
|
|
|
configfile: "config/config.yaml"
|
|
|
|
|
2024-07-11 15:17:16 +02:00
|
|
|
include: "utils.smk"
|
|
|
|
|
|
|
|
import datetime
|
|
|
|
DATE = datetime.datetime.now().strftime("%Y%m%d")
|
|
|
|
|
2024-07-19 16:33:27 +02:00
|
|
|
ARTIFACTS_FOLDER_NICKEL = config["folder_artifacts_nickel"]
|
|
|
|
ARTIFACTS_FOLDER_JSON = config["folder_artifacts_json"]
|
|
|
|
BLACKLIST_FOLDER = config["folder_blacklists"]
|
|
|
|
BLACKLIST = config["symlink_blacklist"]
|
2024-07-16 13:59:44 +02:00
|
|
|
EXTENSION = "json"
|
2024-07-20 15:41:56 +02:00
|
|
|
SYSTEM = config["system"]
|
2024-07-11 15:17:16 +02:00
|
|
|
|
2024-07-16 13:59:44 +02:00
|
|
|
ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER_NICKEL, BLACKLIST)
|
2024-07-11 15:17:16 +02:00
|
|
|
|
|
|
|
rule all:
|
|
|
|
input:
|
|
|
|
expand("{folder}/{artifact}/{date}.csv",\
|
2024-07-16 17:35:09 +02:00
|
|
|
folder=["pkgs", "build_status", "artifact_hash"],\
|
2024-07-11 15:17:16 +02:00
|
|
|
artifact=ARTIFACTS,\
|
2024-07-16 16:47:17 +02:00
|
|
|
date=DATE
|
|
|
|
),
|
2024-07-16 17:35:09 +02:00
|
|
|
expand("{folder}/{artifact}/{date}.txt",\
|
|
|
|
folder=["logs"],\
|
|
|
|
artifact=ARTIFACTS,\
|
|
|
|
date=DATE
|
|
|
|
),
|
2024-07-11 15:17:16 +02:00
|
|
|
f"{BLACKLIST_FOLDER}/{DATE}.csv"
|
|
|
|
|
2024-07-19 16:33:27 +02:00
|
|
|
rule check_all:
|
|
|
|
input:
|
|
|
|
expand(f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.json", artifact=ARTIFACTS)
|
|
|
|
|
|
|
|
|
2024-07-16 13:59:44 +02:00
|
|
|
rule check_artifact:
|
|
|
|
input:
|
|
|
|
"flake.nix",
|
|
|
|
"flake.lock",
|
|
|
|
contract="workflow/nickel/artifact_contract.ncl",
|
|
|
|
artifact=f"{ARTIFACTS_FOLDER_NICKEL}/{{artifact}}.ncl"
|
|
|
|
output:
|
|
|
|
f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.json"
|
|
|
|
shell:
|
|
|
|
"""
|
|
|
|
nickel export --format json --output {output} <<< 'let {{Artifact, ..}} = import "{input.contract}" in ((import "{input.artifact}") | Artifact)'
|
|
|
|
"""
|
|
|
|
|
2024-07-20 15:41:56 +02:00
|
|
|
SHELLS_ECG = {
|
|
|
|
"local": f"python3 {{input.ecg}} -l {{output.log}} -p {{output.pkg}} -b {{output.build_status}} -a {{output.artifact_hash}} {ARTIFACTS_FOLDER_JSON}/{{wildcards.artifact}}.{EXTENSION}",
|
|
|
|
"g5k": f"python3 {{input.execo_wrapper}} --site {config['site']} --cluster {config['cluster']} --max_duration {config['max_duration']} --checkpoint {config['checkpoint']} {'--besteffort' if config['besteffort'] else ''} --sleep_time {config['sleep_time']} -- "
|
|
|
|
}
|
|
|
|
|
2024-07-11 15:17:16 +02:00
|
|
|
rule run_ecg:
|
|
|
|
input:
|
|
|
|
"flake.nix",
|
|
|
|
"flake.lock",
|
|
|
|
ecg="ecg.py",
|
2024-07-20 15:41:56 +02:00
|
|
|
execo_wrapper="workflow/scripts/submission_g5k.py",
|
|
|
|
oar_wrapper="workflow/scripts/ecg_wrapper.oar.bash",
|
2024-07-16 13:59:44 +02:00
|
|
|
artifact=f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.{EXTENSION}"
|
2024-07-11 15:17:16 +02:00
|
|
|
output:
|
2024-07-16 16:47:17 +02:00
|
|
|
log = "logs/{artifact}/{date}.txt",
|
2024-07-11 15:17:16 +02:00
|
|
|
pkg = "pkgs/{artifact}/{date}.csv",
|
2024-07-16 16:47:17 +02:00
|
|
|
build_status = "build_status/{artifact}/{date}.csv",
|
|
|
|
artifact_hash = "artifact_hash/{artifact}/{date}.csv"
|
2024-07-11 15:17:16 +02:00
|
|
|
shell:
|
2024-07-20 15:41:56 +02:00
|
|
|
(SHELLS_ECG["g5k"] if SYSTEM == "g5k" else "") + SHELLS_ECG["local"]
|
2024-07-11 15:17:16 +02:00
|
|
|
|
|
|
|
rule update_blacklist:
|
|
|
|
input:
|
|
|
|
BLACKLIST,
|
2024-07-16 16:47:17 +02:00
|
|
|
build_status=expand("build_status/{artifact}/{{date}}.csv",\
|
2024-07-11 15:17:16 +02:00
|
|
|
artifact=ARTIFACTS)
|
|
|
|
output:
|
|
|
|
f"{BLACKLIST_FOLDER}/{{date}}.csv"
|
|
|
|
shell:
|
|
|
|
f"cat {{input}} > {{output}} && ln -s {{output}} {BLACKLIST}"
|