configfile: "config/config.yaml" include: "utils.smk" import os import datetime DATE = datetime.datetime.now().strftime("%Y%m%d") ARTIFACTS_FOLDER_NICKEL = config["folder_artifacts_nickel"] ARTIFACTS_FOLDER_JSON = config["folder_artifacts_json"] BLACKLIST_FOLDER = config["folder_blacklists"] BLACKLIST = config["symlink_blacklist"] EXTENSION = "json" SYSTEM = config["system"] PREFIX = config["prefix"] ANALYSIS_DIR = config["analysis_dir"] ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER_NICKEL, BLACKLIST) rule all: input: expand(f"{PREFIX}/{{folder}}/{{artifact}}/{{date}}.csv",\ folder=["pkgs", "build_status", "artifact_hash"],\ artifact=ARTIFACTS,\ date=DATE ), expand(f"{PREFIX}/{{folder}}/{{artifact}}/{{date}}.txt",\ folder=["logs"],\ artifact=ARTIFACTS,\ date=DATE ), f"{BLACKLIST_FOLDER}/{DATE}.csv" rule check_all: input: expand(f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.json", artifact=ARTIFACTS) rule check_artifact: input: "flake.nix", "flake.lock", contract="workflow/nickel/artifact_contract.ncl", artifact=f"{ARTIFACTS_FOLDER_NICKEL}/{{artifact}}.ncl" output: f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.json" shell: """ nickel export --format json --output {output} <<< 'let {{Artifact, ..}} = import "{input.contract}" in ((import "{input.artifact}") | Artifact)' """ SHELLS_ECG = { "local": f"./{{input.ecg_wrapper}} {{input.ecg}} {ARTIFACTS_FOLDER_JSON}/{{wildcards.artifact}}.{EXTENSION} {{output.pkg}} {{output.build_status}} {{output.artifact_hash}} {{output.log}}", "g5k": f"python3 {{input.execo_wrapper}} --path {os.getcwd()} --script {{input.oar_wrapper}} --site {config['site']} --cluster {config['cluster']} --max-duration {config['max_duration']} --checkpoint {config['checkpoint']} {'--besteffort' if config['besteffort'] else ''} --sleep_time {config['sleep_time']} --build_status_file {{output.build_status}} --artifact {{wildcards.artifact}} -- '" } rule run_ecg: input: "flake.nix", "flake.lock", ecg="ecg.py", ecg_wrapper="workflow/scripts/ecg_wrapper.sh", execo_wrapper="workflow/scripts/submission_g5k.py", oar_wrapper="workflow/scripts/ecg_oar_wrapper.oar.bash", artifact=f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.{EXTENSION}" output: log = f"{PREFIX}/logs/{{artifact}}/{{date}}.txt", pkg = f"{PREFIX}/pkgs/{{artifact}}/{{date}}.csv", build_status = f"{PREFIX}/build_status/{{artifact}}/{{date}}.csv", artifact_hash = f"{PREFIX}/artifact_hash/{{artifact}}/{{date}}.csv" shell: (SHELLS_ECG["g5k"] if SYSTEM == "g5k" else "") + SHELLS_ECG["local"] + ("'" if SYSTEM == "g5k" else "") rule update_blacklist: input: BLACKLIST, build_status=expand(f"{PREFIX}/build_status/{{artifact}}/{{{{date}}}}.csv",\ artifact=ARTIFACTS ) output: f"{BLACKLIST_FOLDER}/{{date}}.csv" shell: # We need to ignore lines where build is successful: f"cat {{input}} | grep -v ',success' > {{output}} && rm -rf {BLACKLIST} && ln -s {{output}} {BLACKLIST}" ANALYSIS_SCRIPTS = { "sources_stats": f"python3 {ANALYSIS_DIR}/softenv_analysis.py -t sources-stats", "pkgs_changes": f"python3 {ANALYSIS_DIR}/softenv_analysis.py -t pkgs-changes", "build_status": f"python3 {ANALYSIS_DIR}/buildstatus_analysis.py", "artifact": f"python3 {ANALYSIS_DIR}/artifact_analysis.py" } rule all_analysis: input: expand(f"{PREFIX}/{{folder}}/{{artifact}}/{{date}}.csv",\ folder=["pkgs", "build_status", "artifact_hash"],\ artifact=ARTIFACTS,\ date=DATE ), output: expand(f"{ANALYSIS_DIR}/{{folder}}/{{artifact}}/{{date}}.csv",\ folder=["sources_stats", "pkgs_changes", "build_status", "artifact"],\ artifact=ARTIFACTS,\ date=DATE ), shell: f"{{{{ANALYSIS_SCRIPTS[wildcards.folder]}}}} -i {{input}} -o {{output}}"