GuilloteauQ/study-docker-repro-longevity
1
0
Fork 0
Code Issues 6 Pull Requests Actions Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/workflow_config'

Browse Source
This commit is contained in:
antux18 2024-07-23 17:26:03 +02:00
commit 687b777912
17 changed files with 203 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -3,7 +3,7 @@ output/*
cache/*
examples/*
.snakemake/*
artifacts_json/*
artifacts/json/*
pkglist.csv
log.txt
build_status.csv
build_status.csv

3
artifacts_nickel/breitweiser_high.ncl → artifacts/nickel/breitweiser_high.ncl
View File

@ -1,4 +1,3 @@
let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "https://zenodo.org/record/7544675/files/SF2-code.tar.gz",
type = "tar",
@ -28,4 +27,4 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
url = "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/paraview_v5.9.0_ubuntu-20.04_default.tar.gz"
},
],
} | Artifact
}

3
artifacts_nickel/isakov_taxonomy.ncl → artifacts/nickel/isakov_taxonomy.ncl
View File

@ -1,4 +1,3 @@
let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "https://zenodo.org/record/6632461/files/SC22_artifacts_submission.zip",
type = "zip",
@ -9,4 +8,4 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
package_managers = [ "dpkg", "pip" ],
git_packages = [],
misc_packages = [],
} | Artifact
}

3
artifacts_nickel/moses_high.ncl → artifacts/nickel/moses_high.ncl
View File

@ -1,4 +1,3 @@
let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "https://zenodo.org/record/7508499/files/wsmoses/PolygeistGPU-Docker-v0.2.1.zip",
type = "zip",
@ -13,4 +12,4 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
misc_packages = [
{ name = "cmake-3.23.1", url = "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1.tar.gz" }
],
} | Artifact
}

3
artifacts_nickel/parasyris_approximate.ncl → artifacts/nickel/parasyris_approximate.ncl
View File

@ -1,4 +1,3 @@
let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "https://zenodo.org/record/6926481/files/koparasy/HPAC-v0.0.0-Puppeteer.zip",
type = "zip",
@ -11,4 +10,4 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
misc_packages = [
{ name = "HPAC", url = "https://github.com/koparasy/HPAC/archive/refs/heads/develop.zip" }
],
} | Artifact
}

3
artifacts_nickel/template.ncl → artifacts/nickel/template.ncl
View File

@ -1,4 +1,3 @@
let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "https://example.com/artifact.zip",
type = "zip",
@ -13,4 +12,4 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
misc_packages = [
{ name = "mpkg1", url = "http://example.com/package.zip" }
]
} | Artifact
}

3
artifacts_nickel/test.ncl → artifacts/nickel/test.ncl
View File

@ -1,4 +1,3 @@
let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "http://localhost/artifact.zip",
type = "zip",
@ -13,4 +12,4 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
misc_packages = [
{ name = "mpkg1", url = "http://localhost/package1.zip" }
],
} | Artifact
}

3
artifacts_nickel/wang_tgopt.ncl → artifacts/nickel/wang_tgopt.ncl
View File

@ -1,4 +1,3 @@
let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "https://zenodo.org/record/7328505/files/tgopt-artifact.tgz",
type = "tar",
@ -11,4 +10,4 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
misc_packages = [
{name = "Miniconda3-py37_4.12.0-Linux-x86_64", url = "https://repo.anaconda.com/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh" }
],
} | Artifact
}

3
artifacts_nickel/ziogas_deinsum-cpu.ncl → artifacts/nickel/ziogas_deinsum-cpu.ncl
View File

@ -1,4 +1,3 @@
let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "https://zenodo.org/record/7004393/files/deinsum/sc22-artifact-0.4.zip",
type = "zip",
@ -22,4 +21,4 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
url = "https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz"
}
],
} | Artifact
}

3
artifacts_nickel/ziogas_deinsum-gpu.ncl → artifacts/nickel/ziogas_deinsum-gpu.ncl
View File

@ -1,4 +1,3 @@
let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "https://zenodo.org/record/7004393/files/deinsum/sc22-artifact-0.4.zip",
type = "zip",
@ -15,4 +14,4 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
misc_packages = [
{ name = "pip", url = "https://bootstrap.pypa.io/get-pip.py" }
],
} | Artifact
}

14
config/config.yaml Normal file
View File

@ -0,0 +1,14 @@
folder_artifacts_nickel: "artifacts/nickel"
folder_artifacts_json: "artifacts/json"
folder_blacklists: "blacklists"
symlink_blacklist: "blacklist.csv"
system: "g5k" # can be "local" for local execution
prefix: "outputs"
site: "grenoble"
cluster: "dahu"
max_duration: 60 # 1 hour
checkpoint: 1 # 1 minute
besteffort: True
#sleep_time: 300 # 5 minutes
sleep_time: 30 # 0.5 minutes

19
ecg.py
View File

@ -68,7 +68,7 @@ def download_file(url, dest):
hash_process = subprocess.run(f"sha256sum {file.name} | cut -d ' ' -f 1 | tr -d '\n'", capture_output=True, shell=True)
return hash_process.stdout.decode("utf-8")
def download_sources(config, arthashlog_path, cachedir_path, use_cache):
def download_sources(config, arthashlog_path, tmp_dir, use_cache):
"""
Downloads the source of the artifact in 'config'.
@ -93,7 +93,7 @@ def download_sources(config, arthashlog_path, cachedir_path, use_cache):
"""
url = config["artifact_url"]
artifact_name = trim(url)
artifact_dir = os.path.join(cachedir_path, artifact_name)
artifact_dir = os.path.join(tmp_dir, artifact_name)
# Checking if artifact in cache. Not downloading if it is:
if not os.path.exists(artifact_dir) or not use_cache:
logging.info(f"Downloading artifact from {url}")
@ -150,6 +150,7 @@ def buildstatus_saver(output, buildstatus_path, config_path):
file_exists = os.path.exists(buildstatus_path)
buildstatus_file = open(buildstatus_path, "a")
artifact_name = os.path.basename(config_path).split(".")[0]
# # Writing header in case file didn't exist:
# if not file_exists:
# buildstatus_file.write("yaml_path,timestamp,error")
@ -159,11 +160,12 @@ def buildstatus_saver(output, buildstatus_path, config_path):
unknown_error = False
now = datetime.datetime.now()
timestamp = str(datetime.datetime.timestamp(now))
buildstatus_file.write(f"{config_path},{timestamp},{error_cat}\n")
buildstatus_file.write(f"{artifact_name},{timestamp},{error_cat}\n")
print(unknown_error)
if unknown_error:
now = datetime.datetime.now()
timestamp = str(datetime.datetime.timestamp(now))
buildstatus_file.write(f"{config_path},{timestamp},unknown_error\n")
buildstatus_file.write(f"{artifact_name},{timestamp},unknown_error\n")
buildstatus_file.close()
def build_image(config, src_dir):
@ -361,12 +363,15 @@ def main():
# print(config)
config_file.close()
src_dir = download_sources(config, arthashlog_path, cachedir_path, use_cache)
return_code, build_output = build_image(config, src_dir)
tmp_dir = tempfile.TemporaryDirectory()
artifact_dir = download_sources(config, arthashlog_path, tmp_dir.name, use_cache)
return_code, build_output = build_image(config, artifact_dir)
if return_code == 0:
check_env(config, src_dir, pkglist_path)
check_env(config, artifact_dir, pkglist_path)
remove_image(config)
pathlib.Path(buildstatus_path).touch()
else:
pathlib.Path(pkglist_path).touch()
buildstatus_saver(build_output, buildstatus_path, config_path)
if not use_cache:

55
flake.lock generated
View File

@ -18,6 +18,45 @@
"type": "github"
}
},
"flake-utils_2": {
"inputs": {
"systems": "systems_2"
},
"locked": {
"lastModified": 1689068808,
"narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"kapack": {
"inputs": {
"flake-utils": "flake-utils_2",
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1718633852,
"narHash": "sha256-KVeKDdab2wMYMo60mEQHz8Dus4ddhxJ1HPCXzUt9ei8=",
"owner": "oar-team",
"repo": "nur-kapack",
"rev": "052fb35eb29228d9e4ea8afa09e9f0e390782cbd",
"type": "github"
},
"original": {
"owner": "oar-team",
"repo": "nur-kapack",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1701282334,
@ -37,6 +76,7 @@
"root": {
"inputs": {
"flake-utils": "flake-utils",
"kapack": "kapack",
"nixpkgs": "nixpkgs"
}
},
@ -54,6 +94,21 @@
"repo": "default",
"type": "github"
}
},
"systems_2": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",

7
flake.nix
View File

@ -4,12 +4,15 @@
inputs = {
nixpkgs.url = "github:nixos/nixpkgs/23.11";
flake-utils.url = "github:numtide/flake-utils";
kapack.url = "github:oar-team/nur-kapack";
kapack.inputs.nixpkgs.follows = "nixpkgs";
};
outputs = { self, nixpkgs, flake-utils }:
outputs = { self, nixpkgs, flake-utils, kapack }:
flake-utils.lib.eachDefaultSystem (system:
let
pkgs = import nixpkgs { inherit system; };
kapkgs = kapack.packages.${system};
in
{
devShells = {
@ -18,8 +21,10 @@
snakemake
gawk
nickel
# TODO separate into several shells
(python3.withPackages (ps: with ps; [
requests
kapkgs.execo
]))
];
};

43
workflow/Snakefile
View File

@ -1,30 +1,40 @@
configfile: "config/config.yaml"
include: "utils.smk"
import os
import datetime
DATE = datetime.datetime.now().strftime("%Y%m%d")
ARTIFACTS_FOLDER_NICKEL = "artifacts_nickel"
ARTIFACTS_FOLDER_JSON = "artifacts_json"
BLACKLIST_FOLDER = "blacklists"
BLACKLIST = "blacklist.csv"
ARTIFACTS_FOLDER_NICKEL = config["folder_artifacts_nickel"]
ARTIFACTS_FOLDER_JSON = config["folder_artifacts_json"]
BLACKLIST_FOLDER = config["folder_blacklists"]
BLACKLIST = config["symlink_blacklist"]
EXTENSION = "json"
SYSTEM = config["system"]
PREFIX = config["prefix"]
ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER_NICKEL, BLACKLIST)
rule all:
input:
expand("{folder}/{artifact}/{date}.csv",\
expand(f"{PREFIX}/{{folder}}/{{artifact}}/{{date}}.csv",\
folder=["pkgs", "build_status", "artifact_hash"],\
artifact=ARTIFACTS,\
date=DATE
),
expand("{folder}/{artifact}/{date}.txt",\
expand(f"{PREFIX}/{{folder}}/{{artifact}}/{{date}}.txt",\
folder=["logs"],\
artifact=ARTIFACTS,\
date=DATE
),
f"{BLACKLIST_FOLDER}/{DATE}.csv"
rule check_all:
input:
expand(f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.json", artifact=ARTIFACTS)
rule check_artifact:
input:
"flake.nix",
@ -38,26 +48,33 @@ rule check_artifact:
nickel export --format json --output {output} <<< 'let {{Artifact, ..}} = import "{input.contract}" in ((import "{input.artifact}") | Artifact)'
"""
SHELLS_ECG = {
"local": f"python3 {{input.ecg}} -l {{output.log}} -p {{output.pkg}} -b {{output.build_status}} -a {{output.artifact_hash}} {ARTIFACTS_FOLDER_JSON}/{{wildcards.artifact}}.{EXTENSION}",
"g5k": f"python3 {{input.execo_wrapper}} --path {os.getcwd()} --script {{input.oar_wrapper}} --site {config['site']} --cluster {config['cluster']} --max-duration {config['max_duration']} --checkpoint {config['checkpoint']} {'--besteffort' if config['besteffort'] else ''} --sleep_time {config['sleep_time']} --build_status_file {{output.build_status}} --artifact {{wildcards.artifact}} -- '"
}
rule run_ecg:
input:
"flake.nix",
"flake.lock",
ecg="ecg.py",
execo_wrapper="workflow/scripts/submission_g5k.py",
oar_wrapper="workflow/scripts/ecg_wrapper.oar.bash",
artifact=f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.{EXTENSION}"
output:
log = "logs/{artifact}/{date}.txt",
pkg = "pkgs/{artifact}/{date}.csv",
build_status = "build_status/{artifact}/{date}.csv",
artifact_hash = "artifact_hash/{artifact}/{date}.csv"
log = f"{PREFIX}/logs/{{artifact}}/{{date}}.txt",
pkg = f"{PREFIX}/pkgs/{{artifact}}/{{date}}.csv",
build_status = f"{PREFIX}/build_status/{{artifact}}/{{date}}.csv",
artifact_hash = f"{PREFIX}/artifact_hash/{{artifact}}/{{date}}.csv"
shell:
f"python3 {{input.ecg}} -l {{output.log}} -p {{output.pkg}} -b {{output.build_status}} -a {{output.artifact_hash}} {ARTIFACTS_FOLDER_JSON}/{{wildcards.artifact}}.{EXTENSION}"
(SHELLS_ECG["g5k"] if SYSTEM == "g5k" else "") + SHELLS_ECG["local"] + ("'" if SYSTEM == "g5k" else "")
rule update_blacklist:
input:
BLACKLIST,
build_status=expand("build_status/{artifact}/{{date}}.csv",\
build_status=expand(f"{PREFIX}/build_status/{{artifact}}/{{{{date}}}}.csv",\
artifact=ARTIFACTS)
output:
f"{BLACKLIST_FOLDER}/{{date}}.csv"
shell:
f"cat {{input}} > {{output}} && ln -s {{output}} {BLACKLIST}"
f"cat {{input}} > {{output}} && rm -rf {BLACKLIST} && ln -s {{output}} {BLACKLIST}"

25
workflow/scripts/ecg_wrapper.oar.bash Executable file
View File

@ -0,0 +1,25 @@
#!/bin/bash
set -xe
DIRECTORY=$1
shift
BUILD_STATUS_FILE=$1
shift
ARTIFACT_FILE=$1
shift
# To "activate" nix on the node
export PATH=~/.local/bin:$PATH
# Install Docker on the node (-t is to store the images on /tmp because it has more disk)
# https://www.grid5000.fr/w/Docker
g5k-setup-docker -t
handler() {
echo "${ARTIFACT_FILE}, `date +%s.%N`, exceeded_time" >> ${BUILD_STATUS_FILE}; exit 0;
}
trap handler SIGUSR2
cd ${DIRECTORY}
nix develop --command $@

50
workflow/scripts/submission_g5k.py Normal file
View File

@ -0,0 +1,50 @@
from execo_g5k import oardel, oarsub, OarSubmission, wait_oar_job_start, get_oar_job_nodes, get_oar_job_info
import time
import argparse
def submit_job(cluster, site, maximum_duration_minutes, checkpoint_minutes, is_besteffort, path, script, command, build_status_file, artifact):
reservation_duration = (maximum_duration_minutes + checkpoint_minutes) * 60
checkpoint = checkpoint_minutes * 60
job_type = []
if is_besteffort:
job_type.append("besteffort")
oar_job_id, _site = oarsub([(OarSubmission(f"{{cluster='{cluster}'}}/nodes=1",\
reservation_duration,\
job_type=job_type,\
additional_options=f"--checkpoint {checkpoint}",\
command=f"{path}/{script} {path} {build_status_file} {artifact} {command}"), site)])[0]
return oar_job_id
def wait_for_completion(oar_job_id, site, sleep_time):
state = "Running"
while state != "Terminated" and state != "Error":
time.sleep(sleep_time)
info = get_oar_job_info(oar_job_id, site)
state = info["state"]
def main():
parser = argparse.ArgumentParser(description="Wrapper script to submit to OAR from a namespace")
parser.add_argument("--site", required=True, help="Grid'5000 site to submit to")
parser.add_argument("--cluster", required=True, help="Cluster to submit to")
parser.add_argument("--max-duration", required=True, type=int, help="Max Duration in MINUTES of the docker build")
parser.add_argument("--checkpoint", required=True, type=int, help="Duration in MINUTES before the end of the job to do the checkpoint")
parser.add_argument("--besteffort", action='store_false', help="Submit the job as besteffort")
parser.add_argument("--path", required=True, help="Root of the project")
parser.add_argument("--script", required=True, help="Path of the bash script to oarsub relative to the '--path'")
parser.add_argument("--sleep_time", required=False, type=int, default=60, help="Time interval in seconds to check the termination of the job")
parser.add_argument("--build_status_file", required=True, help="File to write the build status to in the case of time exceeding")
parser.add_argument("--artifact", required=True, help="Name of the artifact")
parser.add_argument("command", help="ECG Command")
args = parser.parse_args()
oar_job_id = submit_job(args.cluster, args.site, args.max_duration, args.checkpoint, args.besteffort, args.path, args.script, args.command, args.build_status_file, args.artifact)
wait_oar_job_start(oar_job_id, args.site)
wait_for_completion(oar_job_id, args.site, args.sleep_time)
return 0
main()