From cc50349cf54e254692382288f8498be649a6dc4b Mon Sep 17 00:00:00 2001 From: antux18 Date: Tue, 23 Jul 2024 21:04:34 +0200 Subject: [PATCH] Almost solved the removing of bash login messages from outputs (#18) but still needs some work ("dace" pip package incorrectly displayed for deinsum). --- artifacts/nickel/moses_high.ncl | 2 +- artifacts/nickel/shi_welder.ncl | 20 +++++++++++++++ artifacts/nickel/template.ncl | 2 +- artifacts/nickel/test.ncl | 2 +- artifacts/nickel/wang_tgopt.ncl | 2 +- artifacts/nickel/ziogas_deinsum-cpu.ncl | 6 ++--- ecg.py | 34 +++++++++++++++---------- 7 files changed, 48 insertions(+), 20 deletions(-) create mode 100644 artifacts/nickel/shi_welder.ncl diff --git a/artifacts/nickel/moses_high.ncl b/artifacts/nickel/moses_high.ncl index 10f2f04..c1fa1f1 100644 --- a/artifacts/nickel/moses_high.ncl +++ b/artifacts/nickel/moses_high.ncl @@ -7,7 +7,7 @@ dockerfile_location = "wsmoses-PolygeistGPU-Docker-ba18197/MocCUDA", package_managers = [ "dpkg", "pip" ], git_packages = [ - { name = "MocCUDA", location = "/root/MocCUDA"} + { name = "MocCUDA", location = "/root/MocCUDA" } ], misc_packages = [ { name = "cmake-3.23.1", url = "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1.tar.gz" } diff --git a/artifacts/nickel/shi_welder.ncl b/artifacts/nickel/shi_welder.ncl new file mode 100644 index 0000000..9fc728c --- /dev/null +++ b/artifacts/nickel/shi_welder.ncl @@ -0,0 +1,20 @@ +{ + artifact_url = "https://github.com/microsoft/nnfusion/archive/refs/heads/osdi2023welder.zip", + type = "zip", + doi = "", + comment = "", + image_name = "welder_cuda", + dockerfile_location = "nnfusion-osdi2023welder", + package_managers = [ "dpkg", "pip", "conda" ], + git_packages = [ + { name = "tvm", location = "/tvm" }, + { name = "nnfusion", location = "/nnfusion" }, + { name = "cutlass", location = "/cutlass" } + ], + misc_packages = [ + { + name = "Miniconda3-py310_23.1.0-1-Linux-x86_64", + url = "https://repo.anaconda.com/miniconda/Miniconda3-py310_23.1.0-1-Linux-x86_64.sh" + } + ] +} diff --git a/artifacts/nickel/template.ncl b/artifacts/nickel/template.ncl index 069a66c..d13322b 100644 --- a/artifacts/nickel/template.ncl +++ b/artifacts/nickel/template.ncl @@ -7,7 +7,7 @@ dockerfile_location = "path/to/docker/folder", package_managers = [ "dpkg" ], git_packages = [ - { name = "pkg1", location = "path/to/git/repo"} + { name = "pkg1", location = "path/to/git/repo" } ], misc_packages = [ { name = "mpkg1", url = "http://example.com/package.zip" } diff --git a/artifacts/nickel/test.ncl b/artifacts/nickel/test.ncl index 589bffa..68d261c 100644 --- a/artifacts/nickel/test.ncl +++ b/artifacts/nickel/test.ncl @@ -7,7 +7,7 @@ dockerfile_location = "./", package_managers = [ "dpkg", "pip" ], git_packages = [ - { name = "pkg1", location = "/pkg1"} + { name = "pkg1", location = "/pkg1" } ], misc_packages = [ { name = "mpkg1", url = "http://localhost/package1.zip" } diff --git a/artifacts/nickel/wang_tgopt.ncl b/artifacts/nickel/wang_tgopt.ncl index 65b673c..2cd04e6 100644 --- a/artifacts/nickel/wang_tgopt.ncl +++ b/artifacts/nickel/wang_tgopt.ncl @@ -8,6 +8,6 @@ package_managers = [ "dpkg", "conda" ], git_packages = [], misc_packages = [ - {name = "Miniconda3-py37_4.12.0-Linux-x86_64", url = "https://repo.anaconda.com/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh" } + { name = "Miniconda3-py37_4.12.0-Linux-x86_64", url = "https://repo.anaconda.com/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh" } ], } diff --git a/artifacts/nickel/ziogas_deinsum-cpu.ncl b/artifacts/nickel/ziogas_deinsum-cpu.ncl index 8407d06..c471c97 100644 --- a/artifacts/nickel/ziogas_deinsum-cpu.ncl +++ b/artifacts/nickel/ziogas_deinsum-cpu.ncl @@ -7,9 +7,9 @@ dockerfile_location = "deinsum-sc22-artifact-7559901/docker_cpu", package_managers = [ "dpkg", "pip" ], git_packages = [ - { name = "dace", location = "/dace"}, - { name = "multilinear-algebra-sc22-artifact", location = "/multilinear-algebra-sc22-artifact"}, - { name = "ctf", location = "/ctf"} + { name = "dace", location = "/dace" }, + { name = "multilinear-algebra-sc22-artifact", location = "/multilinear-algebra-sc22-artifact" }, + { name = "ctf", location = "/ctf" } ], misc_packages = [ { diff --git a/ecg.py b/ecg.py index 14ae9e5..1c4b16d 100755 --- a/ecg.py +++ b/ecg.py @@ -219,38 +219,46 @@ def check_env(config, src_dir, pkglist_path): # Commands to list installed packages along with their versions and the name # of the package manager, depending on the package managers. # Each package manager is associated with a tuple, the first item being - # the query command, and the second being the command that will format + # the package manager's command, the second being the arguments for the + # query (they must be separate for the "--entrypoint" argument of Docker + # 'run', see below), and the third one being the command that will format # the output of the query command (this one can be an empty string in case # the formatting part is already done using the options of the first command). - # The first needs to be run on the container, and the second on the host, - # to take into account container images that do not have the formatting + # The first command needs to be run on the container, and the second on the + # host, to take into account container images that do not have the formatting # packages installed. pkgmgr_cmd = { - "dpkg": ("dpkg -l", "awk 'NR>5 {print $2 \",\" $3 \",\" \"dpkg\"}'"), \ - "rpm":("rpm -qa --queryformat '%{NAME},%{VERSION},rpm\\n'", ""), \ - "pacman":("pacman -Q", "awk '{print $0 \",\" $1 \",pacman\"}'"), \ - "pip":("pip freeze", "sed 's/==/,/g' | awk '{print $0 \",pip\"}'"), \ - "conda":("/root/.conda/bin/conda list -e", "sed 's/=/ /g' | awk 'NR>3 {print $1 \",\" $2 \",conda\"}'") + "dpkg": ("dpkg", "-l", "awk 'NR>5 {print $2 \",\" $3 \",\" \"dpkg\"}'"), \ + "rpm":("rpm", "-qa --queryformat '%{NAME},%{VERSION},rpm\\n'", ""), \ + "pacman":("pacman", "-Q", "awk '{print $0 \",\" $1 \",pacman\"}'"), \ + "pip":("pip", "freeze", "sed 's/==/,/g' | awk '{print $0 \",pip\"}'"), \ + "conda":("/root/.conda/bin/conda", "list -e", "sed 's/=/ /g' | awk 'NR>3 {print $1 \",\" $2 \",conda\"}'") } - # Command to obtain the latest commit hash in a git repository: - gitcmd = "git log -n 1 --pretty=format:%H" + # Command to obtain the latest commit hash in a git repository (separated + # into 2 parts for "--entrypoint"): + gitcmd = ("git", "log -n 1 --pretty=format:%H") logging.info("Checking software environment") pkglist_file = open(pkglist_path, "w") # pkglist_file.write("package,version,package_manager\n") path = os.path.join(src_dir, config["dockerfile_location"]) for pkgmgr in config["package_managers"]: + # "--entrypoint" requires command and arguments to be separated. + # This Docker 'run' option is used to prevent the shell from printing + # a login message, if any. pkglist_cmd = pkgmgr_cmd[pkgmgr][0] - listformat_cmd = pkgmgr_cmd[pkgmgr][1] + pkglist_cmdargs = pkgmgr_cmd[pkgmgr][1].split(" ") + listformat_cmd = pkgmgr_cmd[pkgmgr][2] logging.info(f"Checking '{pkgmgr}'") - pkglist_process = subprocess.run(["docker", "run", "--rm", config["image_name"]] + pkglist_cmd.split(" "), cwd=path, capture_output=True) + # pkglist_process = subprocess.run(["docker", "run", "--rm", config["image_name"]] + pkglist_cmd.split(" "), cwd=path, capture_output=True) + pkglist_process = subprocess.run(["docker", "run", "--rm", "--entrypoint", pkglist_cmd, config["image_name"]] + pkglist_cmdargs, cwd=path, capture_output=True) format_process = subprocess.run(f"cat << EOF | {listformat_cmd}\n{pkglist_process.stdout.decode('utf-8')}EOF", cwd=path, capture_output=True, shell=True) pkglist = format_process.stdout.decode("utf-8") pkglist_file.write(pkglist) if "git_packages" in config.keys(): logging.info("Checking Git packages") for repo in config["git_packages"]: - pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", repo["location"], config["image_name"]] + gitcmd.split(" "), cwd=path, capture_output=True) + pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", repo["location"], "--entrypoint", gitcmd[0], config["image_name"]] + gitcmd[1].split(" "), cwd=path, capture_output=True) repo_row = f"{repo['name']},{pkglist_process.stdout.decode('utf-8')},git" pkglist_file.write(f"{repo_row}\n") if "misc_packages" in config.keys():