GuilloteauQ/study-docker-repro-longevity
1
0
Fork 0
Code Issues 6 Pull Requests Actions Projects Releases Wiki Activity

Written Nickel + JSON artifact config files from their YAML version. Handling unknown errors during image building.

Browse Source
This commit is contained in:
antux18 2024-07-18 17:56:57 +02:00
parent d791064183
commit 57ff34e8b9
19 changed files with 328 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
artifacts_json/breitweiser_high.json Normal file
View File

@ -0,0 +1,38 @@
{
"artifact_url": "https://zenodo.org/record/7544675/files/SF2-code.tar.gz",
"dockerfile_location": "bdm-paper-examples/docker",
"doi": "10.1145/3572848.3577480",
"git_packages": [
{
"location": "~/.pyenv",
"name": "pyenv"
}
],
"image_name": "bdm-publication-image-v7",
"misc_packages": [
{
"name": "NetLogo-6.2.0-64",
"type": "tar",
"url": "http://ccl.northwestern.edu/netlogo/6.2.0/NetLogo-6.2.0-64.tgz"
},
{
"name": "root_v6.22.06_python3.9_ubuntu-20.04",
"type": "tar",
"url": "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/root_v6.22.06_python3.9_ubuntu-20.04.tar.gz"
},
{
"name": "qt_v5.12.10_ubuntu-20.04",
"type": "tar",
"url": "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/qt_v5.12.10_ubuntu-20.04.tar.gz"
},
{
"name": "paraview_v5.9.0_ubuntu-20.04_default",
"type": "tar",
"url": "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/paraview_v5.9.0_ubuntu-20.04_default.tar.gz"
}
],
"package_managers": [
"dpkg"
],
"type": "tar"
}

13
artifacts_json/isakov_taxonomy.json Normal file
View File

@ -0,0 +1,13 @@
{
"artifact_url": "https://zenodo.org/record/6632461/files/SC22_artifacts_submission.zip",
"dockerfile_location": "SC22_artifacts_submission",
"doi": "10.5555/3571885.3571906",
"git_packages": [],
"image_name": "taxo:artefact",
"misc_packages": [],
"package_managers": [
"dpkg",
"pip"
],
"type": "zip"
}

24
artifacts_json/moses_high.json Normal file
View File

@ -0,0 +1,24 @@
{
"artifact_url": "https://zenodo.org/record/7508499/files/wsmoses/PolygeistGPU-Docker-v0.2.1.zip",
"dockerfile_location": "wsmoses-PolygeistGPU-Docker-ba18197/MocCUDA",
"doi": "10.1145/3572848.3577475",
"git_packages": [
{
"location": "/root/MocCUDA",
"name": "MocCUDA"
}
],
"image_name": "polygeist:artefact",
"misc_packages": [
{
"name": "cmake-3.23.1",
"type": "tar",
"url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1.tar.gz"
}
],
"package_managers": [
"dpkg",
"pip"
],
"type": "zip"
}

19
artifacts_json/parasyris_approximate.json Normal file
View File

@ -0,0 +1,19 @@
{
"artifact_url": "https://zenodo.org/record/6926481/files/koparasy/HPAC-v0.0.0-Puppeteer.zip",
"dockerfile_location": "koparasy-HPAC-2723bb8/approx/puppeteer/container",
"doi": "10.5555/3571885.3571974",
"git_packages": [],
"image_name": "puppeteer:artefact",
"misc_packages": [
{
"name": "HPAC",
"type": "zip",
"url": "https://github.com/koparasy/HPAC/archive/refs/heads/develop.zip"
}
],
"package_managers": [
"dpkg",
"pip"
],
"type": "zip"
}

0
artifacts_json/example.json → artifacts_json/template.json
View File

19
artifacts_json/wang_tgopt.json Normal file
View File

@ -0,0 +1,19 @@
{
"artifact_url": "https://zenodo.org/record/7328505/files/tgopt-artifact.tgz",
"dockerfile_location": "tgopt-artifact",
"doi": "10.1145/3572848.3577490",
"git_packages": [],
"image_name": "tgopt:artefact",
"misc_packages": [
{
"name": "Miniconda3-py37_4.12.0-Linux-x86_64",
"type": "zip",
"url": "https://repo.anaconda.com/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh"
}
],
"package_managers": [
"dpkg",
"conda"
],
"type": "tar"
}

37
artifacts_json/ziogas_deinsum-cpu.json Normal file
View File

@ -0,0 +1,37 @@
{
"artifact_url": "https://zenodo.org/record/7004393/files/deinsum/sc22-artifact-0.4.zip",
"dockerfile_location": "deinsum-sc22-artifact-7559901/docker_cpu",
"doi": "10.5555/3571885.3571918",
"git_packages": [
{
"location": "/dace",
"name": "dace"
},
{
"location": "/multilinear-algebra-sc22-artifact",
"name": "multilinear-algebra-sc22-artifact"
},
{
"location": "/ctf",
"name": "ctf"
}
],
"image_name": "deinsum:artefact-cpu",
"misc_packages": [
{
"name": "mpich-3.1.4",
"type": "tar",
"url": "http://www.mpich.org/static/downloads/3.1.4/mpich-3.1.4.tar.gz"
},
{
"name": "Python-3.10.2",
"type": "tar",
"url": "https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz"
}
],
"package_managers": [
"dpkg",
"pip"
],
"type": "zip"
}

32
artifacts_json/ziogas_deinsum-gpu.json Normal file
View File

@ -0,0 +1,32 @@
{
"artifact_url": "https://zenodo.org/record/7004393/files/deinsum/sc22-artifact-0.4.zip",
"dockerfile_location": "deinsum-sc22-artifact-7559901/docker_gpu",
"doi": "10.5555/3571885.3571918",
"git_packages": [
{
"location": "/dace",
"name": "dace"
},
{
"location": "/multilinear-algebra-sc22-artifact",
"name": "multilinear-algebra-sc22-artifact"
},
{
"location": "/ctf",
"name": "ctf"
}
],
"image_name": "deinsum:artefact_gpu",
"misc_packages": [
{
"name": "pip",
"type": "zip",
"url": "https://bootstrap.pypa.io/get-pip.py"
}
],
"package_managers": [
"dpkg",
"pip"
],
"type": "zip"
}

34
artifacts_nickel/breitweiser_high.ncl Normal file
View File

@ -0,0 +1,34 @@
let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "https://zenodo.org/record/7544675/files/SF2-code.tar.gz",
type = "tar",
doi = "10.1145/3572848.3577480",
image_name = "bdm-publication-image-v7",
dockerfile_location = "bdm-paper-examples/docker",
package_managers = [ "dpkg" ],
git_packages = [
{ name = "pyenv", location = "~/.pyenv" }
],
misc_packages = [
{
name = "NetLogo-6.2.0-64",
url = "http://ccl.northwestern.edu/netlogo/6.2.0/NetLogo-6.2.0-64.tgz",
type = "tar"
},
{
name = "root_v6.22.06_python3.9_ubuntu-20.04",
url = "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/root_v6.22.06_python3.9_ubuntu-20.04.tar.gz",
type = "tar"
},
{
name = "qt_v5.12.10_ubuntu-20.04",
url = "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/qt_v5.12.10_ubuntu-20.04.tar.gz",
type = "tar"
},
{
name = "paraview_v5.9.0_ubuntu-20.04_default",
url = "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/paraview_v5.9.0_ubuntu-20.04_default.tar.gz",
type = "tar"
},
],
} | Artifact

11
artifacts_nickel/isakov_taxonomy.ncl Normal file
View File

@ -0,0 +1,11 @@
let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "https://zenodo.org/record/6632461/files/SC22_artifacts_submission.zip",
type = "zip",
doi = "10.5555/3571885.3571906",
image_name = "taxo:artefact",
dockerfile_location = "SC22_artifacts_submission",
package_managers = [ "dpkg", "pip" ],
git_packages = [],
misc_packages = [],
} | Artifact

15
artifacts_nickel/moses_high.ncl Normal file
View File

@ -0,0 +1,15 @@
let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "https://zenodo.org/record/7508499/files/wsmoses/PolygeistGPU-Docker-v0.2.1.zip",
type = "zip",
doi = "10.1145/3572848.3577475",
image_name = "polygeist:artefact",
dockerfile_location = "wsmoses-PolygeistGPU-Docker-ba18197/MocCUDA",
package_managers = [ "dpkg", "pip" ],
git_packages = [
{ name = "MocCUDA", location = "/root/MocCUDA"}
],
misc_packages = [
{ name = "cmake-3.23.1", url = "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1.tar.gz", type = "tar" }
],
} | Artifact

13
artifacts_nickel/parasyris_approximate.ncl Normal file
View File

@ -0,0 +1,13 @@
let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "https://zenodo.org/record/6926481/files/koparasy/HPAC-v0.0.0-Puppeteer.zip",
type = "zip",
doi = "10.5555/3571885.3571974",
image_name = "puppeteer:artefact",
dockerfile_location = "koparasy-HPAC-2723bb8/approx/puppeteer/container",
package_managers = [ "dpkg", "pip" ],
git_packages = [],
misc_packages = [
{ name = "HPAC", url = "https://github.com/koparasy/HPAC/archive/refs/heads/develop.zip", type = "zip" } # CMake also but not downloaded, so cannot add to list...
],
} | Artifact

0
artifacts_nickel/example.ncl → artifacts_nickel/template.ncl
View File

3
artifacts_nickel/test.ncl
View File

@ -1,3 +1,4 @@
let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "http://localhost/artifact.zip",
type = "zip",
@ -11,4 +12,4 @@
misc_packages = [
{ name = "mpkg1", url = "http://localhost/package1.zip", type = "zip" }
],
}
} | Artifact

13
artifacts_nickel/wang_tgopt.ncl Normal file
View File

@ -0,0 +1,13 @@
let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "https://zenodo.org/record/7328505/files/tgopt-artifact.tgz",
type = "tar",
doi = "10.1145/3572848.3577490",
image_name = "tgopt:artefact",
dockerfile_location = "tgopt-artifact",
package_managers = [ "dpkg", "conda" ],
git_packages = [],
misc_packages = [
{name = "Miniconda3-py37_4.12.0-Linux-x86_64", url = "https://repo.anaconda.com/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh", type = "zip"}
],
} | Artifact

26
artifacts_nickel/ziogas_deinsum-cpu.ncl Normal file
View File

@ -0,0 +1,26 @@
let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "https://zenodo.org/record/7004393/files/deinsum/sc22-artifact-0.4.zip",
type = "zip",
doi = "10.5555/3571885.3571918",
image_name = "deinsum:artefact-cpu",
dockerfile_location = "deinsum-sc22-artifact-7559901/docker_cpu",
package_managers = [ "dpkg", "pip" ],
git_packages = [
{ name = "dace", location = "/dace"},
{ name = "multilinear-algebra-sc22-artifact", location = "/multilinear-algebra-sc22-artifact"},
{ name = "ctf", location = "/ctf"}
],
misc_packages = [
{
name = "mpich-3.1.4",
url = "http://www.mpich.org/static/downloads/3.1.4/mpich-3.1.4.tar.gz",
type = "tar"
},
{
name = "Python-3.10.2",
url = "https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz",
type = "tar"
}
],
} | Artifact

17
artifacts_nickel/ziogas_deinsum-gpu.ncl Normal file
View File

@ -0,0 +1,17 @@
let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "https://zenodo.org/record/7004393/files/deinsum/sc22-artifact-0.4.zip",
type = "zip",
doi = "10.5555/3571885.3571918",
image_name = "deinsum:artefact_gpu",
dockerfile_location = "deinsum-sc22-artifact-7559901/docker_gpu",
package_managers = [ "dpkg", "pip" ],
git_packages = [
{ name = "dace", location = "/dace"},
{ name = "multilinear-algebra-sc22-artifact", location = "/multilinear-algebra-sc22-artifact"},
{ name = "ctf", location = "/ctf"}
],
misc_packages = [
{ name = "pip", url = "https://bootstrap.pypa.io/get-pip.py", type = "zip"}
],
} | Artifact

2
blacklists/blacklist.csv
View File

@ -1 +1 @@
example, IMAGE_NOT_FOUND, 0
template, IMAGE_NOT_FOUND, 0

1 example template IMAGE_NOT_FOUND 0

20
ecg.py
View File

@ -167,11 +167,18 @@ def buildstatus_saver(output):
# # Writing header in case file didn't exist:
# if not file_exists:
# buildstatus_file.write("yaml_path,timestamp,error")
unknown_error = True
for error_cat, error in build_errors.items():
if error in output:
unknown_error = False
now = datetime.datetime.now()
timestamp = str(datetime.datetime.timestamp(now))
buildstatus_file.write(f"{config_path},{timestamp},{error_cat}\n")
print(unknown_error)
if unknown_error:
now = datetime.datetime.now()
timestamp = str(datetime.datetime.timestamp(now))
buildstatus_file.write(f"{config_path},{timestamp},unknown_error\n")
buildstatus_file.close()
def build_image(config, src_dir):
@ -188,8 +195,8 @@ def build_image(config, src_dir):
Returns
-------
bool
'True' if build successful, 'False' otherwise.
return_code: bool, build_output: str
Return code and output of Docker 'build'.
"""
name = config["image_name"]
logging.info(f"Starting building image {name}")
@ -202,8 +209,7 @@ def build_image(config, src_dir):
logging.info(build_output)
return_code = build_process.returncode
logging.info(f"Command '{build_command}' exited with code {return_code}")
buildstatus_saver(build_output)
return return_code == 0
return return_code, build_output
def check_env(config, src_dir):
"""
@ -342,10 +348,12 @@ def main():
config_file.close()
src_dir = download_sources(config)
successful_build = build_image(config, src_dir)
if successful_build:
return_code, build_output = build_image(config, src_dir)
if return_code == 0:
check_env(config, src_dir)
remove_image(config)
else:
buildstatus_saver(build_output)
if not use_cache:
os.system(f"rm -rf {os.path.join(cachedir_path, trim(config['artifact_url']))}")