study-docker-repro-longevity/workflow/Snakefile

107 lines
3.9 KiB
Plaintext
Raw Normal View History

2024-07-19 16:33:27 +02:00
configfile: "config/config.yaml"
2024-07-11 15:17:16 +02:00
include: "utils.smk"
import os
2024-07-11 15:17:16 +02:00
import datetime
DATE = datetime.datetime.now().strftime("%Y%m%d")
2024-07-19 16:33:27 +02:00
ARTIFACTS_FOLDER_NICKEL = config["folder_artifacts_nickel"]
ARTIFACTS_FOLDER_JSON = config["folder_artifacts_json"]
BLACKLIST_FOLDER = config["folder_blacklists"]
BLACKLIST = config["symlink_blacklist"]
2024-07-16 13:59:44 +02:00
EXTENSION = "json"
2024-07-20 15:41:56 +02:00
SYSTEM = config["system"]
PREFIX = config["prefix"]
ANALYSIS_DIR = config["analysis_dir"]
2024-07-11 15:17:16 +02:00
2024-07-16 13:59:44 +02:00
ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER_NICKEL, BLACKLIST)
2024-07-11 15:17:16 +02:00
rule all:
input:
expand(f"{PREFIX}/{{folder}}/{{artifact}}/{{date}}.csv",\
folder=["pkgs", "build_status", "artifact_hash"],\
2024-07-11 15:17:16 +02:00
artifact=ARTIFACTS,\
date=DATE
),
expand(f"{PREFIX}/{{folder}}/{{artifact}}/{{date}}.txt",\
folder=["logs"],\
artifact=ARTIFACTS,\
date=DATE
),
2024-07-11 15:17:16 +02:00
f"{BLACKLIST_FOLDER}/{DATE}.csv"
2024-07-19 16:33:27 +02:00
rule check_all:
input:
expand(f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.json", artifact=ARTIFACTS)
2024-07-16 13:59:44 +02:00
rule check_artifact:
input:
"flake.nix",
"flake.lock",
contract="workflow/nickel/artifact_contract.ncl",
artifact=f"{ARTIFACTS_FOLDER_NICKEL}/{{artifact}}.ncl"
output:
f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.json"
shell:
"""
nickel export --format json --output {output} <<< 'let {{Artifact, ..}} = import "{input.contract}" in ((import "{input.artifact}") | Artifact)'
"""
2024-07-20 15:41:56 +02:00
SHELLS_ECG = {
"local": f"./{{input.ecg_wrapper}} {{input.ecg}} {ARTIFACTS_FOLDER_JSON}/{{wildcards.artifact}}.{EXTENSION} {{output.pkg}} {{output.build_status}} {{output.artifact_hash}} {{output.log}}",
"g5k": f"python3 {{input.execo_wrapper}} --path {os.getcwd()} --script {{input.oar_wrapper}} --site {config['site']} --cluster {config['cluster']} --max-duration {config['max_duration']} --checkpoint {config['checkpoint']} {'--besteffort' if config['besteffort'] else ''} --sleep_time {config['sleep_time']} --build_status_file {{output.build_status}} --artifact {{wildcards.artifact}} -- '"
2024-07-20 15:41:56 +02:00
}
2024-07-11 15:17:16 +02:00
rule run_ecg:
input:
"flake.nix",
"flake.lock",
ecg="ecg.py",
ecg_wrapper="workflow/scripts/ecg_wrapper.sh",
2024-07-20 15:41:56 +02:00
execo_wrapper="workflow/scripts/submission_g5k.py",
oar_wrapper="workflow/scripts/ecg_oar_wrapper.oar.bash",
2024-07-16 13:59:44 +02:00
artifact=f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.{EXTENSION}"
2024-07-11 15:17:16 +02:00
output:
log = f"{PREFIX}/logs/{{artifact}}/{{date}}.txt",
pkg = f"{PREFIX}/pkgs/{{artifact}}/{{date}}.csv",
build_status = f"{PREFIX}/build_status/{{artifact}}/{{date}}.csv",
artifact_hash = f"{PREFIX}/artifact_hash/{{artifact}}/{{date}}.csv"
2024-07-11 15:17:16 +02:00
shell:
(SHELLS_ECG["g5k"] if SYSTEM == "g5k" else "") + SHELLS_ECG["local"] + ("'" if SYSTEM == "g5k" else "")
2024-07-11 15:17:16 +02:00
rule update_blacklist:
input:
BLACKLIST,
build_status=expand(f"{PREFIX}/build_status/{{artifact}}/{{{{date}}}}.csv",\
artifact=ARTIFACTS
)
2024-07-11 15:17:16 +02:00
output:
f"{BLACKLIST_FOLDER}/{{date}}.csv"
shell:
# We need to ignore lines where build is successful:
f"cat {{input}} | grep -v ',success' > {{output}} && rm -rf {BLACKLIST} && ln -s {{output}} {BLACKLIST}"
ANALYSIS_SCRIPTS = {
"sources_stats": f"python3 {ANALYSIS_DIR}/softenv_analysis.py -t sources-stats",
"pkgs_changes": f"python3 {ANALYSIS_DIR}/softenv_analysis.py -t pkgs-changes",
"build_status": f"python3 {ANALYSIS_DIR}/buildstatus_analysis.py",
"artifact": f"python3 {ANALYSIS_DIR}/artifact_analysis.py"
}
rule all_analysis:
input:
expand(f"{PREFIX}/{{folder}}/{{artifact}}/{{date}}.csv",\
folder=["pkgs", "build_status", "artifact_hash"],\
artifact=ARTIFACTS,\
date=DATE
),
output:
expand(f"{ANALYSIS_DIR}/{{folder}}/{{artifact}}/{{date}}.csv",\
folder=["sources_stats", "pkgs_changes", "build_status", "artifact"],\
artifact=ARTIFACTS,\
date=DATE
),
shell:
f"{{{{ANALYSIS_SCRIPTS[wildcards.folder]}}}} -i {{input}} -o {{output}}"