GuilloteauQ/study-docker-repro-longevity
1
0
Fork 0
Code Issues 6 Pull Requests Actions Projects Releases Wiki Activity

Merge branch 'main' of https://forge.chapril.org/GuilloteauQ/study-docker-repro-longevity

Browse Source
This commit is contained in:
antux18 2024-07-23 09:38:24 +02:00
commit 35f8b4c369
31 changed files with 71 additions and 339 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -3,6 +3,7 @@ output/*
cache/*
examples/*
.snakemake/*
artifacts_json/*
pkglist.csv
log.txt
build_status.csv

38
README.md
View File

@ -38,6 +38,44 @@ Where:
- `<artifact_hash_log>` is the path to the file where to log the hash of the downloaded artifact.
- `<cache_directory>` is the path to the cache directory, where downloaded artifacts will be stored for future usage. If not specified, cache is disabled.
## Output
### Package list
The list of packages installed in the container, depending on the package managers, Git packages and other miscellaneous packages given in the config file, in the form of a CSV file, with the following columns in order:
| Package name | Version | Package manager |
|--------------|---------|-----------------|
For Git packages, the hash of the last commit is used as version number. For miscellaneous packages, the hash of the file that has been used to install the package is used as version number.
### Output log
Just a plain text file containing the output of the script.
### Build status file
The log of the failed attempts to build the Docker image, in the form of a CSV file, with the following columns in order:
| Config file path | Timestamp | Reason category |
|------------------|-----------|-----------------|
The timestamp corresponds to when the error is being logged, not to when it happened.
The following are the categories of reasons explaining why the building failed:
- `package_unavailable`: A command requested the installation of a package that is not available.
- `baseimage_unavailable`: The base image needed for this container is not available.
- `unknown_error`: Any other error.
### Artifact hash log
The log of the hash of the artifact archive file, in the form of a CSV file, with the following columns in order:
| Timestamp | Hash |
|-----------|------|
The timestamp corresponds to when the hash has been logged, not to when the artifact has been downloaded.
## License
This project is licensed under the GNU General Public License version 3. You can find the terms of the license in the file LICENSE.

38
artifacts_json/breitweiser_high.json
View File

@ -1,38 +0,0 @@
{
"artifact_url": "https://zenodo.org/record/7544675/files/SF2-code.tar.gz",
"dockerfile_location": "bdm-paper-examples/docker",
"doi": "10.1145/3572848.3577480",
"git_packages": [
{
"location": "~/.pyenv",
"name": "pyenv"
}
],
"image_name": "bdm-publication-image-v7",
"misc_packages": [
{
"name": "NetLogo-6.2.0-64",
"type": "tar",
"url": "http://ccl.northwestern.edu/netlogo/6.2.0/NetLogo-6.2.0-64.tgz"
},
{
"name": "root_v6.22.06_python3.9_ubuntu-20.04",
"type": "tar",
"url": "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/root_v6.22.06_python3.9_ubuntu-20.04.tar.gz"
},
{
"name": "qt_v5.12.10_ubuntu-20.04",
"type": "tar",
"url": "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/qt_v5.12.10_ubuntu-20.04.tar.gz"
},
{
"name": "paraview_v5.9.0_ubuntu-20.04_default",
"type": "tar",
"url": "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/paraview_v5.9.0_ubuntu-20.04_default.tar.gz"
}
],
"package_managers": [
"dpkg"
],
"type": "tar"
}

13
artifacts_json/isakov_taxonomy.json
View File

@ -1,13 +0,0 @@
{
"artifact_url": "https://zenodo.org/record/6632461/files/SC22_artifacts_submission.zip",
"dockerfile_location": "SC22_artifacts_submission",
"doi": "10.5555/3571885.3571906",
"git_packages": [],
"image_name": "taxo:artefact",
"misc_packages": [],
"package_managers": [
"dpkg",
"pip"
],
"type": "zip"
}

24
artifacts_json/moses_high.json
View File

@ -1,24 +0,0 @@
{
"artifact_url": "https://zenodo.org/record/7508499/files/wsmoses/PolygeistGPU-Docker-v0.2.1.zip",
"dockerfile_location": "wsmoses-PolygeistGPU-Docker-ba18197/MocCUDA",
"doi": "10.1145/3572848.3577475",
"git_packages": [
{
"location": "/root/MocCUDA",
"name": "MocCUDA"
}
],
"image_name": "polygeist:artefact",
"misc_packages": [
{
"name": "cmake-3.23.1",
"type": "tar",
"url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1.tar.gz"
}
],
"package_managers": [
"dpkg",
"pip"
],
"type": "zip"
}

19
artifacts_json/parasyris_approximate.json
View File

@ -1,19 +0,0 @@
{
"artifact_url": "https://zenodo.org/record/6926481/files/koparasy/HPAC-v0.0.0-Puppeteer.zip",
"dockerfile_location": "koparasy-HPAC-2723bb8/approx/puppeteer/container",
"doi": "10.5555/3571885.3571974",
"git_packages": [],
"image_name": "puppeteer:artefact",
"misc_packages": [
{
"name": "HPAC",
"type": "zip",
"url": "https://github.com/koparasy/HPAC/archive/refs/heads/develop.zip"
}
],
"package_managers": [
"dpkg",
"pip"
],
"type": "zip"
}

23
artifacts_json/template.json
View File

@ -1,23 +0,0 @@
{
"artifact_url": "https://example.com/artifact.zip",
"dockerfile_location": "path/to/docker/folder",
"doi": "...",
"git_packages": [
{
"location": "path/to/git/repo",
"name": "pkg1"
}
],
"image_name": "image:version",
"misc_packages": [
{
"name": "mpkg1",
"type": "zip",
"url": "https://"
}
],
"package_managers": [
"dpkg"
],
"type": "zip"
}

24
artifacts_json/test.json
View File

@ -1,24 +0,0 @@
{
"artifact_url": "http://localhost/artifact.zip",
"dockerfile_location": "./",
"doi": "...",
"git_packages": [
{
"location": "/pkg1",
"name": "pkg1"
}
],
"image_name": "prog:latest",
"misc_packages": [
{
"name": "mpkg1",
"type": "zip",
"url": "http://localhost/package1.zip"
}
],
"package_managers": [
"dpkg",
"pip"
],
"type": "zip"
}

19
artifacts_json/wang_tgopt.json
View File

@ -1,19 +0,0 @@
{
"artifact_url": "https://zenodo.org/record/7328505/files/tgopt-artifact.tgz",
"dockerfile_location": "tgopt-artifact",
"doi": "10.1145/3572848.3577490",
"git_packages": [],
"image_name": "tgopt:artefact",
"misc_packages": [
{
"name": "Miniconda3-py37_4.12.0-Linux-x86_64",
"type": "zip",
"url": "https://repo.anaconda.com/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh"
}
],
"package_managers": [
"dpkg",
"conda"
],
"type": "tar"
}

37
artifacts_json/ziogas_deinsum-cpu.json
View File

@ -1,37 +0,0 @@
{
"artifact_url": "https://zenodo.org/record/7004393/files/deinsum/sc22-artifact-0.4.zip",
"dockerfile_location": "deinsum-sc22-artifact-7559901/docker_cpu",
"doi": "10.5555/3571885.3571918",
"git_packages": [
{
"location": "/dace",
"name": "dace"
},
{
"location": "/multilinear-algebra-sc22-artifact",
"name": "multilinear-algebra-sc22-artifact"
},
{
"location": "/ctf",
"name": "ctf"
}
],
"image_name": "deinsum:artefact-cpu",
"misc_packages": [
{
"name": "mpich-3.1.4",
"type": "tar",
"url": "http://www.mpich.org/static/downloads/3.1.4/mpich-3.1.4.tar.gz"
},
{
"name": "Python-3.10.2",
"type": "tar",
"url": "https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz"
}
],
"package_managers": [
"dpkg",
"pip"
],
"type": "zip"
}

32
artifacts_json/ziogas_deinsum-gpu.json
View File

@ -1,32 +0,0 @@
{
"artifact_url": "https://zenodo.org/record/7004393/files/deinsum/sc22-artifact-0.4.zip",
"dockerfile_location": "deinsum-sc22-artifact-7559901/docker_gpu",
"doi": "10.5555/3571885.3571918",
"git_packages": [
{
"location": "/dace",
"name": "dace"
},
{
"location": "/multilinear-algebra-sc22-artifact",
"name": "multilinear-algebra-sc22-artifact"
},
{
"location": "/ctf",
"name": "ctf"
}
],
"image_name": "deinsum:artefact_gpu",
"misc_packages": [
{
"name": "pip",
"type": "zip",
"url": "https://bootstrap.pypa.io/get-pip.py"
}
],
"package_managers": [
"dpkg",
"pip"
],
"type": "zip"
}

13
artifacts_nickel/breitweiser_high.ncl
View File

@ -3,6 +3,7 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
artifact_url = "https://zenodo.org/record/7544675/files/SF2-code.tar.gz",
type = "tar",
doi = "10.1145/3572848.3577480",
comment = "",
image_name = "bdm-publication-image-v7",
dockerfile_location = "bdm-paper-examples/docker",
package_managers = [ "dpkg" ],
@ -12,23 +13,19 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
misc_packages = [
{
name = "NetLogo-6.2.0-64",
url = "http://ccl.northwestern.edu/netlogo/6.2.0/NetLogo-6.2.0-64.tgz",
type = "tar"
url = "http://ccl.northwestern.edu/netlogo/6.2.0/NetLogo-6.2.0-64.tgz"
},
{
name = "root_v6.22.06_python3.9_ubuntu-20.04",
url = "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/root_v6.22.06_python3.9_ubuntu-20.04.tar.gz",
type = "tar"
url = "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/root_v6.22.06_python3.9_ubuntu-20.04.tar.gz"
},
{
name = "qt_v5.12.10_ubuntu-20.04",
url = "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/qt_v5.12.10_ubuntu-20.04.tar.gz",
type = "tar"
url = "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/qt_v5.12.10_ubuntu-20.04.tar.gz"
},
{
name = "paraview_v5.9.0_ubuntu-20.04_default",
url = "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/paraview_v5.9.0_ubuntu-20.04_default.tar.gz",
type = "tar"
url = "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/paraview_v5.9.0_ubuntu-20.04_default.tar.gz"
},
],
} | Artifact

1
artifacts_nickel/isakov_taxonomy.ncl
View File

@ -3,6 +3,7 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
artifact_url = "https://zenodo.org/record/6632461/files/SC22_artifacts_submission.zip",
type = "zip",
doi = "10.5555/3571885.3571906",
comment = "",
image_name = "taxo:artefact",
dockerfile_location = "SC22_artifacts_submission",
package_managers = [ "dpkg", "pip" ],

3
artifacts_nickel/moses_high.ncl
View File

@ -3,6 +3,7 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
artifact_url = "https://zenodo.org/record/7508499/files/wsmoses/PolygeistGPU-Docker-v0.2.1.zip",
type = "zip",
doi = "10.1145/3572848.3577475",
comment = "",
image_name = "polygeist:artefact",
dockerfile_location = "wsmoses-PolygeistGPU-Docker-ba18197/MocCUDA",
package_managers = [ "dpkg", "pip" ],
@ -10,6 +11,6 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{ name = "MocCUDA", location = "/root/MocCUDA"}
],
misc_packages = [
{ name = "cmake-3.23.1", url = "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1.tar.gz", type = "tar" }
{ name = "cmake-3.23.1", url = "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1.tar.gz" }
],
} | Artifact

3
artifacts_nickel/parasyris_approximate.ncl
View File

@ -3,11 +3,12 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
artifact_url = "https://zenodo.org/record/6926481/files/koparasy/HPAC-v0.0.0-Puppeteer.zip",
type = "zip",
doi = "10.5555/3571885.3571974",
comment = "The package 'cmake-3.14.0-Linux-x86_64' is not specified below, because it is installed using a Bash script included in the artifact archive itself.",
image_name = "puppeteer:artefact",
dockerfile_location = "koparasy-HPAC-2723bb8/approx/puppeteer/container",
package_managers = [ "dpkg", "pip" ],
git_packages = [],
misc_packages = [
{ name = "HPAC", url = "https://github.com/koparasy/HPAC/archive/refs/heads/develop.zip", type = "zip" } # CMake also but not downloaded, so cannot add to list...
{ name = "HPAC", url = "https://github.com/koparasy/HPAC/archive/refs/heads/develop.zip" }
],
} | Artifact

3
artifacts_nickel/template.ncl
View File

@ -3,6 +3,7 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
artifact_url = "https://example.com/artifact.zip",
type = "zip",
doi = "...",
comment = "Template for artifact configuration.",
image_name = "image:version",
dockerfile_location = "path/to/docker/folder",
package_managers = [ "dpkg" ],
@ -10,6 +11,6 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{ name = "pkg1", location = "path/to/git/repo"}
],
misc_packages = [
{ name = "mpkg1", url = "http://example.com/package.zip", type = "zip" }
{ name = "mpkg1", url = "http://example.com/package.zip" }
]
} | Artifact

3
artifacts_nickel/test.ncl
View File

@ -3,6 +3,7 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
artifact_url = "http://localhost/artifact.zip",
type = "zip",
doi = "...",
comment = "",
image_name = "prog:latest",
dockerfile_location = "./",
package_managers = [ "dpkg", "pip" ],
@ -10,6 +11,6 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{ name = "pkg1", location = "/pkg1"}
],
misc_packages = [
{ name = "mpkg1", url = "http://localhost/package1.zip", type = "zip" }
{ name = "mpkg1", url = "http://localhost/package1.zip" }
],
} | Artifact

3
artifacts_nickel/wang_tgopt.ncl
View File

@ -3,11 +3,12 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
artifact_url = "https://zenodo.org/record/7328505/files/tgopt-artifact.tgz",
type = "tar",
doi = "10.1145/3572848.3577490",
comment = "",
image_name = "tgopt:artefact",
dockerfile_location = "tgopt-artifact",
package_managers = [ "dpkg", "conda" ],
git_packages = [],
misc_packages = [
{name = "Miniconda3-py37_4.12.0-Linux-x86_64", url = "https://repo.anaconda.com/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh", type = "zip"}
{name = "Miniconda3-py37_4.12.0-Linux-x86_64", url = "https://repo.anaconda.com/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh" }
],
} | Artifact

7
artifacts_nickel/ziogas_deinsum-cpu.ncl
View File

@ -3,6 +3,7 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
artifact_url = "https://zenodo.org/record/7004393/files/deinsum/sc22-artifact-0.4.zip",
type = "zip",
doi = "10.5555/3571885.3571918",
comment = "",
image_name = "deinsum:artefact-cpu",
dockerfile_location = "deinsum-sc22-artifact-7559901/docker_cpu",
package_managers = [ "dpkg", "pip" ],
@ -14,13 +15,11 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
misc_packages = [
{
name = "mpich-3.1.4",
url = "http://www.mpich.org/static/downloads/3.1.4/mpich-3.1.4.tar.gz",
type = "tar"
url = "http://www.mpich.org/static/downloads/3.1.4/mpich-3.1.4.tar.gz"
},
{
name = "Python-3.10.2",
url = "https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz",
type = "tar"
url = "https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz"
}
],
} | Artifact

9
artifacts_nickel/ziogas_deinsum-gpu.ncl
View File

@ -2,16 +2,17 @@ let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
{
artifact_url = "https://zenodo.org/record/7004393/files/deinsum/sc22-artifact-0.4.zip",
type = "zip",
comment = "",
doi = "10.5555/3571885.3571918",
image_name = "deinsum:artefact_gpu",
dockerfile_location = "deinsum-sc22-artifact-7559901/docker_gpu",
package_managers = [ "dpkg", "pip" ],
git_packages = [
{ name = "dace", location = "/dace"},
{ name = "multilinear-algebra-sc22-artifact", location = "/multilinear-algebra-sc22-artifact"},
{ name = "ctf", location = "/ctf"}
{ name = "dace", location = "/dace" },
{ name = "multilinear-algebra-sc22-artifact", location = "/multilinear-algebra-sc22-artifact" },
{ name = "ctf", location = "/ctf" }
],
misc_packages = [
{ name = "pip", url = "https://bootstrap.pypa.io/get-pip.py", type = "zip"}
{ name = "pip", url = "https://bootstrap.pypa.io/get-pip.py" }
],
} | Artifact

7
artifacts_yaml/ppopp23/gpu_to_cpu_transpilation_and_optimization_via_high_level_parallel_constructs.yaml
View File

@ -1,7 +0,0 @@
artifact_url: "https://zenodo.org/record/7508499/files/wsmoses/PolygeistGPU-Docker-v0.2.1.zip"
type: "zip"
doi: "10.1145/3572848.3577475"
image_name: "polygeist:artefact"
dockerfile_location: "wsmoses-PolygeistGPU-Docker-ba18197/MocCUDA"
package_managers:
- "dpkg"

8
artifacts_yaml/ppopp23/high_performance_and_scalable_agent_based_simulation_with_BioDynaMo.yaml
View File

@ -1,8 +0,0 @@
artifact_url: "https://zenodo.org/record/7544675/files/SF2-code.tar.gz"
type: "tar"
doi: "10.1145/3572848.3577480"
image_name: "bdm-publication-image-v7"
dockerfile_location: "bdm-paper-examples/docker"
package_managers:
- "dpkg"
# Build done by running build.sh and not just docker build, but this is not implemented yet

8
artifacts_yaml/ppopp23/tgopt_redundancy_aware-optimizations_for_temporal_graph_attention_network.yaml
View File

@ -1,8 +0,0 @@
artifact_url: "https://zenodo.org/record/7328505/files/tgopt-artifact.tgz"
type: "tar"
doi: "10.1145/3572848.3577490"
image_name: "tgopt:artefact"
dockerfile_location: "tgopt-artifact"
package_managers:
- "dpkg"
- "conda"

8
artifacts_yaml/sc22/a_taxonomy_of_error_sources_in_hpc_io_machine_learning_models.yaml
View File

@ -1,8 +0,0 @@
artifact_url: "https://zenodo.org/record/6632461/files/SC22_artifacts_submission.zip"
type: "zip"
doi: "10.5555/3571885.3571906"
image_name: "taxo:artefact"
dockerfile_location: "SC22_artifacts_submission"
package_managers:
- "dpkg"
- "pip"

8
artifacts_yaml/sc22/approximate_copmuting_through_the_lens_of_uncertainty_quantification.yaml
View File

@ -1,8 +0,0 @@
artifact_url: "https://zenodo.org/record/6926481/files/koparasy/HPAC-v0.0.0-Puppeteer.zip"
type: "zip"
doi: "10.5555/3571885.3571974"
image_name: "puppeteer:artefact"
dockerfile_location: "koparasy-HPAC-2723bb8/approx/puppeteer/container"
package_managers:
- "dpkg"
- "pip"

7
artifacts_yaml/sc22/deinsum_practially_io_optimal_multi_linear_algebra-cpu.yaml
View File

@ -1,7 +0,0 @@
artifact_url: "https://zenodo.org/record/7004393/files/deinsum/sc22-artifact-0.4.zip"
type: "zip"
doi: "10.5555/3571885.3571918"
image_name: "deinsum:artefact-cpu"
dockerfile_location: "deinsum-sc22-artifact-7559901/docker_cpu"
package_managers:
- "dpkg"

7
artifacts_yaml/sc22/deinsum_practially_io_optimal_multi_linear_algebra-gpu.yaml
View File

@ -1,7 +0,0 @@
artifact_url: "https://zenodo.org/record/7004393/files/deinsum/sc22-artifact-0.4.zip"
type: "zip"
doi: "10.5555/3571885.3571918"
image_name: "deinsum:artefact_gpu"
dockerfile_location: "deinsum-sc22-artifact-7559901/docker_gpu"
package_managers:
- "dpkg"

14
artifacts_yaml/template.yaml
View File

@ -1,14 +0,0 @@
artifact_url: "https://example.com/artifact.zip"
type: "zip" # Possible values: zip, tar
doi: "XX.XXXX/XXXXXXX.XXXXXXX"
image_name: "image1:version"
dockerfile_location: "path/to/docker/folder"
package_managers:
- "dpkg" # Possible values: dpkg, rpm, pacman, pip, conda
git_packages:
- name: "pkg1"
location: "path/to/git/repo"
misc_packages:
- name: "mpkg1"
url: "https://example.com/package1.zip"
type: "zip" # Possible values: zip, tar

15
artifacts_yaml/test.yaml
View File

@ -1,15 +0,0 @@
artifact_url: "http://localhost/artifact.zip"
type: "zip"
doi: "XX.XXXX/XXXXXXX.XXXXXXX"
image_name: "prog:latest"
dockerfile_location: "./"
package_managers:
- "dpkg"
- "pip"
git_packages:
- name: "pkg1"
location: "/pkg1"
misc_packages:
- name: "mpkg1"
url: "http://localhost/package1.zip"
type: "zip"

3
blacklists/blacklist.csv
View File

@ -1 +1,2 @@
template, IMAGE_NOT_FOUND, 0
template, unknown_error, 0
test, unknown_error, 0
1 template IMAGE_NOT_FOUND unknown_error 0
2 test unknown_error 0

12
workflow/nickel/artifact_contract.ncl
View File

@ -37,21 +37,21 @@ in
| String,
url
| doc "URL of the package. Will be used to compute the hash"
| URL,
type
| doc "Type of the archive (zip, tar, ...)"
| ArchiveType,
| URL
},
Artifact = {
artifact_url
| doc "URL where to download the artifact"
| URL,
type
| doc "Type of the archive (zip, tar, ...)"
| doc "Type of the archive. Possible values: zip, tar"
| ArchiveType,
doi
| doc "DOI of the artifact"
| String,
comment
| doc "Additional comments on the artifact that couldn't be written elsewhere"
| String,
image_name
| doc "Name to give the image when building"
| ImageName,
@ -59,7 +59,7 @@ in
| doc "Path to the dockerfile in the artifact"
| FilePath,
package_managers
| doc "Package Managers used in the container"
| doc "Package Managers used in the container. Possible values: dpkg, rpm, pacman, pip, conda"
| Array PackageManager,
git_packages
| doc "Git repositories cloned in the container"