diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d4f2a5e --- /dev/null +++ b/.gitignore @@ -0,0 +1,36 @@ +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# vim tags +tags +.tags + +# Docker files +Dockerfile +*.yml diff --git a/README.md b/README.md index 4ddab65..2b0c02a 100644 --- a/README.md +++ b/README.md @@ -1,86 +1,72 @@ # ROCm-docker ### Radeon Open Compute Platform for docker -This repository contains dockerfiles for the various software layers defined in the Radeon Open Compute Platform. Installation instructions for how to install docker on [Ubuntu systems](https://docs.docker.com/v1.8/installation/ubuntulinux/) and [Fedora systems](https://docs.docker.com/v1.8/installation/fedora/) is available. +This repository contains a framework for building the various software layers defined in the Radeon Open Compute Platform into portable docker images. There are docker dependencies to use this framework, which need to be pre-installed on the host. -A bash script `./roc-setup` is provided as a convenience to build various ROC images. The script can receive command line parameters to give it parameters how to build ROCm docker containers. +- Docker on [Ubuntu systems](https://docs.docker.com/v1.8/installation/ubuntulinux/) or [Fedora systems](https://docs.docker.com/v1.8/installation/fedora/) +- [Docker-Compose](https://docs.docker.com/compose/install/) as a highly recommended tool + +At the root of this repository is the bash script `./roc-setup` ```bash Usage: ./roc-setup [--master | --develop] [--release | --debug] +Default flags: --master --release + --master) Build dockerfiles from stable master branches; exclusive with --develop --develop) Build dockerfiles from integration branches; exclusive with --master --release) Build release containers; minimizes size of docker images; exclusive with --debug --debug) Build debug containers; symbols generated and build tree intact for debugging; exclusive with --release -Without explicit parameters, ./roc-setup default flags are --master && --release +Without explicit parameters, `./roc-setup` default flags are --master && --release ``` -The following is an example of images after building both --master and --develop containers. All containers are uniquely named to distinguish how they were built. - -```bash -kknox@machine:~/src/github/ROCm-docker -[develop *% u=] $ docker images -REPOSITORY TAG IMAGE ID CREATED SIZE -roc/hcc-isa-master-release latest d82a7c04f1e4 About an hour ago 1.191 GB -roc/hcc-hsail-master-release latest aa4f0543ad43 About an hour ago 1.441 GB -roc/rocr-dev-release latest 7172446cd45e About an hour ago 732.5 MB -roc/roct-dev-release latest aa711a708039 2 hours ago 666.1 MB -roc/rock-dev latest 7bb9dede7b01 2 hours ago 539 MB -roc/hcc-isa-testing-release latest bd57b41d79fb 11 hours ago 1.191 GB -roc/rocr-master-release latest 954dfa5425fc 12 hours ago 732.6 MB -roc/roct-master-release latest 39399c5cee7d 12 hours ago 666.2 MB -roc/rock-master latest c2e70cf58ea7 12 hours ago 539.1 MB -ubuntu 14.04.3 3876b81b5a81 7 weeks ago 187.9 MB - -Container name decoder: /-- -``` +`./roc-setup` generates Dockerfiles to be consumed by the docker build engine. Each sub-directory of this repository corresponds to a docker 'build context' responsible for a software layer in the ROCm stack. After running the script each directory contains a generated 'Dockerfile'. The parameters to the script control which flavor of the components to build, for instance: debug builds of the /develop branches. | ROC component | | |-----|-----| -| hcc-isa | the compiler that generates GPU ISA out of the backend | -| hcc-hsail | the compiler that generates HSAIL IL out of the backend | +| hcc-isa | the compiler that generates GPU ISA from the backend | +| hcc-hsail | the compiler that generates HSAIL IL from the backend | | rocr | the runtime | | roct | the kernel thunk library | -| rock | the linux kernel with gpu kernel modules | -Even given the existence of these ROC containers, **the ROC kernel has to be installed on the host machine.** This is a design constraint of docker; the linux kernel is not resident in the container. All containers use the host linux kernel, so the host linux kernel must be prepared to support ROC infrastructure. +The ROCm component that can not be used in a docker image is the ROCK-Kernel-Driver[1](#ROCK). In order for the docker framework to function, **the ROCm kernel must be installed on the host machine.** This is a design constraint of docker; the linux kernel is not resident in the container. All containers share the host linux kernel, so the ROCK-Kernel-Driver component must be installed on the host linux kernel. ### Installing ROCK on the host machine. -A [sequence of instructions](https://github.com/RadeonOpenCompute/ROCK-Kernel-Driver#installing-and-configuring-the-kernel) in bash: +A [sequence of instructions](https://github.com/RadeonOpenCompute/ROCK-Kernel-Driver#installing-and-configuring-the-kernel) is provided in the ROCK-Kernel-Driver README. -1. `cd /usr/local/src` -2. `git clone --no-checkout --depth=1 https://github.com/RadeonOpenCompute/ROCK-Kernel-Driver.git` -3. `cd ROCK-Kernel-Driver` -4. `git checkout master -- packages/ubuntu` -5. `dpkg -i packages/ubuntu/*.deb` -6. `echo "KERNEL==\"kfd\", MODE=\"0666\"" | sudo tee /etc/udev/rules.d/kfd.rules` -7. `sudo reboot` +### Docker compose +`./roc-setup` prepares an environment that can be controlled with [Docker Compose](https://docs.docker.com/compose/). An output of the script is a **docker-compose.yml** file in the root of the repository, which coordinates the relationships between the various ROCm software layers. Additionally, the docker-compose.yml file can be extended to easily launch interactive application or development containers built on top of the ROCm software stack. -### Creating an application container -After the ROC software stack has been built, an application can be built in a new container to leverage the ROC stack. The /hcc-project sub-directory contains a template of a new container specifically built of software development. Common and useful development tools are pre-installed into the container to help with software development. To begin the development, simply: -- copy the /hcc-project sub-directory into a new directory name, like /my-roc-project -- open and modify the dockerfile there-in to customize the software - - the template derives from roc/hcc-isa-master-release, but could as easily be changed to roc/hcc-hsail-master-release -- the assumption of the application container is that the developer will map a host directory into the container - - the host directory typically will contain source to compile, such as a git repository - - this makes sure that the source persists after the container closes - - the generated files from the build should be into the users /home directory or /opt - - when the container closes, all generated files are cleaned up and forgotten +### Creating an application/development container +The /rocm-project sub-directory contains a template for a container specifically built for software development. Common and useful development tools are pre-installed into the container. To begin, simply: +- copy the /rocm-project sub-directory into a new directory name, such as /my-rocm-project +- open and customize the Dockerfile; pre-install dependencies and services +- modify the docker-compose.yml.template file to add a new service which launches your new image + - use the existing rocm-project section as an example + - add useful host directories to map into the container +- rerun `./roc-setup` script to generate a new **docker-compose.yml** -### Running an application in the docker stack - -You run the container, and optionally map host directories (for shared source code, for instance) like so: +### Running an application using docker-compose +You run the new container (and its dependencies) with docker-compose. When the container is fully loaded and running, you will be presented with a root prompt within the container. ```bash -docker run -it --rm -v ~/host/project-src:/root/project-src / +docker-compose run --rm ``` | Docker command reference | | |-----|-----| -| docker | docker executable| -| run | docker sub-command | -| -it | attach console to container | -| --rm | when exiting container, delete it | -| -v ~/host/project-src:/root/project-src | map host directory into container | -| user-name/app-name | unique name for container | +| docker-compose | docker compose executable| +| run | sub-command to bring up interactive container | +| --rm | when shutting the container down, delete it | +| my-rocm-project | application service defined in **docker-compose.yml** | -### Todo: -1. Create a proper method to load each software component as a [data-only container](https://docs.docker.com/engine/userguide/containers/dockervolumes/#mount-a-host-directory-as-a-data-volume) +To shut down ROCm dependencies and clean up +```bash +docker-compose down -v +``` +| Docker command reference | | +|-----|-----| +| docker-compose | docker compose executable | +| down | sub-command to shut containers down and remove them | +| -v | clean-up shared volumes | + +### Footnotes: +[1] We actually do provide a container for ROCK-Kernel-Driver, but it not used by the rest of the docker images. It does provide isolation and a reproducible environment for kernel development. diff --git a/docker-compose.yml.template b/docker-compose.yml.template new file mode 100644 index 0000000..45d9324 --- /dev/null +++ b/docker-compose.yml.template @@ -0,0 +1,82 @@ +version: '2' +services: +# The following defines data-only containers +# They are not meant to be used as interactive containers + roct: + build: + context: ./roct + dockerfile: Dockerfile + image: ${roct_name} + container_name: roct + entrypoint: /bin/true + volumes: + - ${roct_volume} + + rocr: + depends_on: + - roct + build: + context: ./rocr + dockerfile: Dockerfile + image: ${rocr_name} + container_name: rocr + entrypoint: /bin/true + volumes: + - ${rocr_volume} + + hcc-isa: + depends_on: + - rocr + build: + context: ./hcc-isa + dockerfile: Dockerfile + image: ${hcc_isa_name} + container_name: hcc-isa + entrypoint: /bin/true + volumes: + - ${hcc_volume} + + hcc-hsail: + depends_on: + - rocr + build: + context: ./hcc-hsail + dockerfile: Dockerfile + image: ${hcc_hsail_name} + container_name: hcc-hsail + entrypoint: /bin/true + volumes: + - ${hcc_volume} + + # The following defines application containers, which depend on the data-only + # containers defiend above to provide their software layers + # These should be run with `docker-compose run --rm ` + rocm-project-isa: + build: + context: ./rocm-project + dockerfile: Dockerfile + devices: + - "/dev/kfd" + image: rocm/rocm-project-isa + # volumes is used to map host directories into the container + volumes: + - ~:/usr/local/src/host-home + volumes_from: + - roct:ro + - rocr:ro + - hcc-isa:ro + + rocm-project-hsail: + build: + context: ./rocm-project + dockerfile: Dockerfile + devices: + - "/dev/kfd" + image: rocm/rocm-project-hsail + # volumes is used to map host directories into the container + volumes: + - ~:/usr/local/src/host-home + volumes_from: + - roct:ro + - rocr:ro + - hcc-hsail:ro diff --git a/hcc-hsail/hcc-hsail-dockerfile.template b/hcc-hsail/hcc-hsail-dockerfile.template index 0207b80..e590b22 100644 --- a/hcc-hsail/hcc-hsail-dockerfile.template +++ b/hcc-hsail/hcc-hsail-dockerfile.template @@ -6,7 +6,7 @@ # This builds the hcc compiler, and depends on an already existing rocr-runtime to be found # ubuntu:14.04.3, the native kernel is '3.19' -FROM ~~rocr_container~~ +FROM ${rocr_name} MAINTAINER Kent Knox # Global environment variables @@ -29,10 +29,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ libtinfo-dev \ libc6-dev-i386 \ gcc-multilib \ - llvm \ llvm-dev \ llvm-runtime \ - libc++1 \ libc++-dev \ libc++abi-dev \ re2c \ @@ -45,26 +43,26 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ rm -rf /var/lib/apt/lists/* # App specific environment variables -ENV HCC_INSTALL_PATH=/opt/hcc HCC_BUILD_PATH=/usr/local/src/hcc-hsail +ENV HCC_INSTALL_PATH=${hcc_volume} HCC_BUILD_PATH=/usr/local/src/hcc-hsail ENV PATH ${PATH}:${HCC_INSTALL_PATH}/bin # Build hcc, install to /usr/local RUN mkdir -p ${HCC_BUILD_PATH} && \ cd ${HCC_BUILD_PATH} && \ - git clone --depth=1 --branch=~~branch~~ https://github.com/RadeonOpenCompute/hcc.git ./ && \ + git clone --depth=1 --branch=${repo_branch_hcc_hsail} https://github.com/RadeonOpenCompute/hcc.git ./ && \ mkdir -p build && \ cd build && \ cmake \ -DCMAKE_INSTALL_PREFIX=${HCC_INSTALL_PATH} \ - -DCMAKE_BUILD_TYPE=~~config~~ \ + -DCMAKE_BUILD_TYPE=${build_config} \ -DHSA_HEADER_DIR=${ROCR_INSTALL_PATH}/include \ -DHSA_LIBRARY_DIR=${ROCR_INSTALL_PATH}/lib \ - -DHSA_KMT_LIBRARY_DIR=${HSATHK_INSTALL_PATH} \ + -DHSA_KMT_LIBRARY_DIR=${HSATHK_INSTALL_PATH}/lib/x86_64-linux-gnu \ .. && \ make -j $(nproc) world && \ make install && \ - cd ~ && rm -rf ${HCC_BUILD_PATH} && \ + ${hcc_hsail_cleanup} \ echo "${HCC_INSTALL_PATH}/lib" >> /etc/ld.so.conf.d/hcc.conf && \ ldconfig -# VOLUME [${HCC_INSTALL_PATH}] +COPY setenv-hcc.sh ${HCC_INSTALL_PATH} diff --git a/hcc-hsail/setenv-hcc.sh b/hcc-hsail/setenv-hcc.sh new file mode 100755 index 0000000..3d28db4 --- /dev/null +++ b/hcc-hsail/setenv-hcc.sh @@ -0,0 +1,37 @@ +HCCPATH=/opt/hcc +HCCVER=3.5.0 + +echo "Appending hcc tools into PATH" +if [ -z "${PATH}" ]; then + export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${HCCPATH}/bin +else + export PATH=${PATH}:${HCCPATH}/bin +fi + +echo "Registering hcc libraries with loader" +echo "${HCCPATH}/lib" >> /etc/ld.so.conf.d/hcc-isa.conf +ldconfig + +# if [ -z "${CPATH}" ]; then +# export CPATH=${HCCPATH}/include +# else +# export CPATH=${CPATH}:${HCCPATH}/include +# fi + +# if [ -z "${LIBRARY_PATH}" ]; then +# export LIBRARY_PATH=${HCCPATH}/lib +# else +# export LIBRARY_PATH=${LIBRARY_PATH}:${HCCPATH}/lib +# fi + +# if [ -z "${MANPATH}" ]; then +# export MANPATH=${HCCPATH}/man +# else +# export MANPATH=${HCCPATH}/man:$MANPATH +# fi + +# if [ -z "${INFOPATH}" ]; then +# export INFOPATH=${HCCPATH}/info +# else +# export INFOPATH=${HCCPATH}/info:$INFOPATH +# fi diff --git a/hcc-isa/hcc-isa-dockerfile.template b/hcc-isa/hcc-isa-dockerfile.template index aa39af0..1e2c47e 100644 --- a/hcc-isa/hcc-isa-dockerfile.template +++ b/hcc-isa/hcc-isa-dockerfile.template @@ -6,7 +6,7 @@ # This builds the hcc compiler, and depends on an already existing rocr-runtime to be found # ubuntu:14.04.3, the native kernel is '3.19' -FROM ~~rocr_container~~ +FROM ${rocr_name} MAINTAINER Kent Knox # Global environment variables @@ -34,7 +34,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ rm -rf /var/lib/apt/lists/* # App specific environment variables; modify path to add hcc and repo commands -ENV HCC_INSTALL_PATH=/opt/hcc HCC_BUILD_PATH=/usr/local/src/hcc-isa +ENV HCC_INSTALL_PATH=${hcc_volume} HCC_BUILD_PATH=/usr/local/src/hcc-isa ENV PATH=${PATH}:${HCC_INSTALL_PATH}/bin:~/bin # Compiling hcc-lc requires a custom build tool @@ -45,14 +45,14 @@ RUN mkdir -p ~/bin && \ # Build hcc-isa, install to /usr/local RUN mkdir -p ${HCC_BUILD_PATH} && \ cd ${HCC_BUILD_PATH} && \ - ~/bin/repo init -u https://github.com/RadeonOpenCompute/HCC-Native-GCN-ISA.git -b ~~branch~~ && \ + ~/bin/repo init --depth=1 -u https://github.com/RadeonOpenCompute/HCC-Native-GCN-ISA.git -b ${repo_branch_hcc_isa} && \ ~/bin/repo sync && \ cd llvm && \ mkdir -p build && \ cd build && \ cmake \ -DCMAKE_INSTALL_PREFIX=${HCC_INSTALL_PATH} \ - -DCMAKE_BUILD_TYPE=~~config~~ \ + -DCMAKE_BUILD_TYPE=${build_config} \ -DLLVM_TARGETS_TO_BUILD="AMDGPU;X86" .. && \ make -j $(nproc) install && \ @@ -61,7 +61,7 @@ RUN mkdir -p ${HCC_BUILD_PATH} && \ cd build && \ cmake \ -DCMAKE_INSTALL_PREFIX=${HCC_INSTALL_PATH} \ - -DCMAKE_BUILD_TYPE=~~config~~ \ + -DCMAKE_BUILD_TYPE=${build_config} \ -DLLVM_DIR="${HCC_BUILD_PATH}/llvm/build" .. && \ make -j $(nproc) install && \ @@ -70,15 +70,15 @@ RUN mkdir -p ${HCC_BUILD_PATH} && \ cd build && \ cmake \ -DCMAKE_INSTALL_PREFIX=${HCC_INSTALL_PATH} \ - -DCMAKE_BUILD_TYPE=~~config~~ \ + -DCMAKE_BUILD_TYPE=${build_config} \ -DHSA_LLVM_BIN_DIR=${HCC_INSTALL_PATH}/bin \ -DHSA_AMDGPU_GPU_TARGET=fiji \ -DAMDPHDRS_DIR=${HCC_BUILD_PATH}/llvm-amdgpu-assembler-extra/build/amdphdrs/ \ -DHSA_USE_AMDGPU_BACKEND=ON .. && \ make -j $(nproc) world && \ make install && \ - rm -rf ${HCC_BUILD_PATH} && \ + ${hcc_isa_cleanup} \ echo "${HCC_INSTALL_PATH}/lib" >> /etc/ld.so.conf.d/hcc-isa.conf && \ ldconfig -# VOLUME [${HCC_INSTALL_PATH}] + COPY setenv-hcc.sh ${HCC_INSTALL_PATH} diff --git a/hcc-isa/setenv-hcc.sh b/hcc-isa/setenv-hcc.sh new file mode 100644 index 0000000..e70d362 --- /dev/null +++ b/hcc-isa/setenv-hcc.sh @@ -0,0 +1,37 @@ +HCC_PATH=/opt/hcc +HCC_VER=3.5.0 + +echo "Appending hcc tools into PATH" +if [ -z "${PATH}" ]; then + export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${HCC_PATH}/bin +else + export PATH=${PATH}:${HCC_PATH}/bin +fi + +echo "Registering hcc libraries with loader" +echo "${HCC_PATH}/lib" >> /etc/ld.so.conf.d/hcc-isa.conf +ldconfig + +# if [ -z "${CPATH}" ]; then +# export CPATH=${HCC_PATH}/include +# else +# export CPATH=${CPATH}:${HCC_PATH}/include +# fi + +# if [ -z "${LIBRARY_PATH}" ]; then +# export LIBRARY_PATH=${HCC_PATH}/lib +# else +# export LIBRARY_PATH=${LIBRARY_PATH}:${HCC_PATH}/lib +# fi + +# if [ -z "${MANPATH}" ]; then +# export MANPATH=${HCC_PATH}/man +# else +# export MANPATH=${HCC_PATH}/man:$MANPATH +# fi + +# if [ -z "${INFOPATH}" ]; then +# export INFOPATH=${HCC_PATH}/info +# else +# export INFOPATH=${HCC_PATH}/info:$INFOPATH +# fi diff --git a/roc-setup.sh b/roc-setup.sh index 4b0e83f..0095e90 100755 --- a/roc-setup.sh +++ b/roc-setup.sh @@ -18,12 +18,6 @@ build_release=1 # Build debug binaries; this leaves build tree intact for greater debugging; exclusive with --release build_debug= -# Build debug binaries; this leaves build tree intact for greater debugging; exclusive with --release -remove_images= - -# Build debug binaries; this leaves build tree intact for greater debugging; exclusive with --release -dry_run= - # ################################################# # helper functions # ################################################# @@ -37,8 +31,6 @@ function display_help() echo "--develop) Build dockerfiles from integration branches; exclusive with --master" echo "--release) Build release containers; minimizes size of docker images; exclusive with --debug" echo "--debug) Build debug containers; symbols generated and build tree intact for debugging; exclusive with --release" - echo "--remove_images) Based on the other flags passed, remove the docker images instead of building them" - echo "--dry_run) Print out what would happen with the script, without executing commands" } # ################################################# @@ -62,12 +54,6 @@ while :; do build_release= build_debug=1 ;; - --remove_images) - remove_images=1 - ;; - --dry_run) - dry_run=1 - ;; -h|--help) display_help exit @@ -79,13 +65,13 @@ while :; do shift done -# hcc-hsail does not have a 'standard' develop branch per-se -repo_branch_hcc_hsail="master" +# hcc-hsail does not have a develop branch +export repo_branch_hcc_hsail="master" -# hcc-isa conforms to a non git-flow branch naming scheme -repo_branch_hcc_isa= +# hcc-isa conforms to a non git-flow naming scheme, 'master' changes the most +export repo_branch_hcc_isa= -repo_branch= +export repo_branch= if [ -n "${build_master}" ]; then repo_branch="master" repo_branch_hcc_isa="testing" @@ -94,85 +80,56 @@ else repo_branch_hcc_isa="master" fi -build_config= +export build_config= +export build_config_roct= +export roct_cleanup= +export rocr_cleanup= +export hcc_hsail_cleanup= +export hcc_isa_cleanup= +export rock_name= +export roct_name= +export rocr_name= +export hcc_hsail_name= +export hcc_isa_name= + +rocm_prefix="rocm/" if [ -n "${build_release}" ]; then - build_config="Release" + build_config='Release' + + rock_name="${rocm_prefix}rock-${repo_branch}" + roct_name="${rocm_prefix}roct-${repo_branch}" + rocr_name="${rocm_prefix}rocr-${repo_branch}" + hcc_hsail_name="${rocm_prefix}hcc-hsail-${repo_branch_hcc_hsail}" + hcc_isa_name="${rocm_prefix}hcc-isa-${repo_branch_hcc_isa}" + + # Custom commands to clean up build directories for each component + # This is to keep release images as small as possible + build_config_roct='REL=1' + roct_cleanup='cd ~ && rm -rf ${HSATHK_BUILD_PATH} &&' + rocr_cleanup='cd ~ && rm -rf ${ROCR_BUILD_PATH} &&' + hcc_hsail_cleanup='cd ~ && rm -rf ${HCC_BUILD_PATH} &&' + hcc_isa_cleanup='cd ~ && rm -rf ${HCC_BUILD_PATH} &&' else - build_config="Debug" + build_config='Debug' + + # For debug builds, name the images as 'debug' + # The comma operator in ${build_config} makes the first letter lower case + rock_name="${rocm_prefix}rock-${repo_branch}" + roct_name="${rocm_prefix}roct-${repo_branch}-${build_config,}" + rocr_name="${rocm_prefix}rocr-${repo_branch}-${build_config,}" + hcc_hsail_name="${rocm_prefix}hcc-hsail-${repo_branch_hcc_hsail}-${build_config,}" + hcc_isa_name="${rocm_prefix}hcc-isa-${repo_branch_hcc_isa}-${build_config,}" fi -# The comma operator in ${build_config} makes the first letter lower case -rock_name="roc/rock-${repo_branch}" -roct_name="roc/roct-${repo_branch}-${build_config,}" -rocr_name="roc/rocr-${repo_branch}-${build_config,}" -hcc_hsail_name="roc/hcc-hsail-${repo_branch_hcc_hsail}-${build_config,}" -hcc_isa_name="roc/hcc-isa-${repo_branch_hcc_isa}-${build_config,}" - -rock_docker_build="cat rock/rock-deb-dockerfile.template | sed s/~~branch~~/${repo_branch}/g -" -roct_docker_build="cat roct/roct-thunk-dockerfile.template | sed s/~~branch~~/${repo_branch}/g - | sed s#~~rock_container~~#${rock_name}#g -" -rocr_docker_build="cat rocr/rocr-make-dockerfile.template | sed s/~~branch~~/${repo_branch}/g - | sed s#~~roct_container~~#${roct_name}#g -" -hcc_hsail_docker_build="cat hcc-hsail/hcc-hsail-dockerfile.template | sed s/~~branch~~/${repo_branch_hcc_hsail}/g - | sed s#~~rocr_container~~#${rocr_name}#g -" -hcc_isa_docker_build="cat hcc-isa/hcc-isa-dockerfile.template | sed s/~~branch~~/${repo_branch_hcc_isa}/g - | sed s#~~rocr_container~~#${rocr_name}#g -" - -# ROCT customization -if [ -n "${build_release}" ]; then - roct_docker_build="${roct_docker_build} | sed s/~~config~~/REL\=1/g -" -else - roct_docker_build="${roct_docker_build} | sed s/~~config~~//g -" -fi - -rocr_docker_build="${rocr_docker_build} | sed s/~~config~~/${build_config}/g -" -hcc_hsail_docker_build="${hcc_hsail_docker_build} | sed s/~~config~~/${build_config}/g -" -hcc_isa_docker_build="${hcc_isa_docker_build} | sed s/~~config~~/${build_config}/g -" +export roct_volume='/opt/roct/' +export rocr_volume='/opt/hsa/' +export hcc_volume='/opt/hcc/' # Uncomment below to print dockerfiles with template substitutions; debugging -#eval ${rock_docker_build} -#eval ${roct_docker_build} -#eval ${rocr_docker_build} -#eval ${hcc_hsail_docker_build} -#eval ${hcc_isa_docker_build} +cat rock/rock-deb-dockerfile.template | envsubst '${repo_branch}' > rock/Dockerfile +cat roct/roct-thunk-dockerfile.template | envsubst '${rock_name}:${repo_branch}:${build_config_roct}:${roct_cleanup}:${roct_volume}' > roct/Dockerfile +cat rocr/rocr-make-dockerfile.template | envsubst '${roct_name}:${repo_branch}:${build_config}:${rocr_cleanup}:${rocr_volume}' > rocr/Dockerfile +cat hcc-hsail/hcc-hsail-dockerfile.template | envsubst '${rocr_name}:${repo_branch_hcc_hsail}:${build_config}:${hcc_hsail_cleanup}:${hcc_volume}' > hcc-hsail/Dockerfile +cat hcc-isa/hcc-isa-dockerfile.template | envsubst '${rocr_name}:${repo_branch_hcc_isa}:${build_config}:${hcc_isa_cleanup}:${hcc_volume}' > hcc-isa/Dockerfile -# Build or remove docker images based on passed in option -if [ -n "${remove_images}" ]; then - rock_docker_build="docker rmi ${rock_name}" - roct_docker_build="docker rmi ${roct_name}" - rocr_docker_build="docker rmi ${rocr_name}" - hcc_hsail_docker_build="docker rmi ${hcc_hsail_name}" - hcc_isa_docker_build="docker rmi ${hcc_isa_name}" -else - rock_docker_build="${rock_docker_build} | docker build -t ${rock_name} -" - roct_docker_build="${roct_docker_build} | docker build -t ${roct_name} -" - rocr_docker_build="${rocr_docker_build} | docker build -t ${rocr_name} -" - hcc_hsail_docker_build="${hcc_hsail_docker_build} | docker build -t ${hcc_hsail_name} -" - hcc_isa_docker_build="${hcc_isa_docker_build} | docker build -t ${hcc_isa_name} -" -fi - -# These statements below generate the actual docker images -if [ -n "${dry_run}" ]; then - echo ${rock_docker_build} - echo ${roct_docker_build} - echo ${rocr_docker_build} - echo ${hcc_hsail_docker_build} - echo ${hcc_isa_docker_build} -else - echo "# #################################################" - echo "# Building ROCK container" - echo "# #################################################" - eval ${rock_docker_build} - echo "# #################################################" - echo "# Building ROCT container" - echo "# #################################################" - eval ${roct_docker_build} - echo "# #################################################" - echo "# Building ROCR container" - echo "# #################################################" - eval ${rocr_docker_build} - echo "# #################################################" - echo "# Building HCC-HSAIL container" - echo "# #################################################" - eval ${hcc_hsail_docker_build} - echo "# #################################################" - echo "# Building HCC-ISA container" - echo "# #################################################" - eval ${hcc_isa_docker_build} -fi +cat docker-compose.yml.template | envsubst '${hcc_isa_name}:${hcc_hsail_name}:${rocr_name}:${roct_name}:${hcc_volume}:${rocr_volume}:${roct_volume}' > docker-compose.yml diff --git a/rock/rock-deb-dockerfile.template b/rock/rock-deb-dockerfile.template index c60bc3d..8dfd0ed 100644 --- a/rock/rock-deb-dockerfile.template +++ b/rock/rock-deb-dockerfile.template @@ -35,8 +35,8 @@ ENV ROCK_BUILD_PATH=/usr/local/src/ROCK-Kernel-Driver # Keep the cloned repository as record of what was compiled into the container, but it does make the container size bigger RUN mkdir -p ${ROCK_BUILD_PATH} && \ cd ${ROCK_BUILD_PATH} && \ - git clone --no-checkout --depth=1 --branch=~~branch~~ https://github.com/RadeonOpenCompute/ROCK-Kernel-Driver.git ./ && \ - git checkout ~~branch~~ -- packages/ubuntu && \ + git clone --no-checkout --depth=1 --branch=${repo_branch} https://github.com/RadeonOpenCompute/ROCK-Kernel-Driver.git ./ && \ + git checkout ${repo_branch} -- packages/ubuntu && \ DEBIAN_FRONTEND=noninteractive dpkg -i packages/ubuntu/compute-*.deb && \ DEBIAN_FRONTEND=noninteractive dpkg -i packages/ubuntu/linux-*.deb && \ cd ~ && rm -rf ${ROCK_BUILD_PATH} && \ diff --git a/rock/rock-make-dockerfile b/rock/rock-make-dockerfile index ab87515..c216a43 100644 --- a/rock/rock-make-dockerfile +++ b/rock/rock-make-dockerfile @@ -35,7 +35,7 @@ ENV ROCK_BUILD_PATH=/usr/local/src/ROCK-Kernel-Driver RUN mkdir -p ${ROCK_BUILD_PATH} && \ cd ${ROCK_BUILD_PATH} && \ - git clone --no-checkout --depth=1 --branch=dev https://github.com/RadeonOpenCompute/ROCK-Kernel-Driver.git ./ && \ + git clone --no-checkout --depth=1 --branch=${repo_branch} https://github.com/RadeonOpenCompute/ROCK-Kernel-Driver.git ./ && \ git checkout dev && \ make mrproper && \ echo "KERNEL==\"kfd\", MODE=\"0666\"" | sudo tee /etc/udev/rules.d/kfd.rules diff --git a/hcc-project/Dockerfile b/rocm-project/Dockerfile similarity index 80% rename from hcc-project/Dockerfile rename to rocm-project/Dockerfile index f687619..fa23eb4 100644 --- a/hcc-project/Dockerfile +++ b/rocm-project/Dockerfile @@ -12,12 +12,12 @@ # 'docker run -it --rm -v [host/directory]:[container/directory]:ro /'. # Example: 'docker run -it --rm -v ~/src/my-hcc-project:/root/my-hcc-project:ro kknox/my-hcc-project'. -# ubuntu:14.04.3, the native kernel is '3.19' -FROM roc/hcc-isa-master-release +# The application container can inherit from any other OS container +FROM ubuntu:14.04 MAINTAINER Kent Knox -# Global environment variables -ENV WORKPATH /opt/my-hcc-project +# Change WORKPATH to a location where +ENV WORKPATH /opt/my-rocm-project # The working directory is meant to be where build files are generated WORKDIR ${WORKPATH} @@ -30,6 +30,7 @@ CMD ["-l"] RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ build-essential \ clang-3.5 \ + libelf1 \ curl \ git \ gdb \ @@ -44,4 +45,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ # 2. Aliases GDB to use its native TUI mode by default RUN curl https://j.mp/spf13-vim3 -L | bash && \ echo "\nalias gdb='gdb --tui'" >> ~/.bashrc && \ - mkdir -p /opt/my-hcc-project + echo "\nsource /opt/roct/setenv-roct.sh" >> ~/.bashrc && \ + echo "\nsource /opt/hsa/setenv-rocr.sh" >> ~/.bashrc && \ + echo "\nsource /opt/hcc/setenv-hcc.sh" >> ~/.bashrc && \ + mkdir -p ${WORKPATH} diff --git a/rocr/rocr-make-dockerfile.template b/rocr/rocr-make-dockerfile.template index 7b444c6..355eb25 100644 --- a/rocr/rocr-make-dockerfile.template +++ b/rocr/rocr-make-dockerfile.template @@ -6,7 +6,7 @@ # This builds the radeon open compute runtime # ubuntu:14.04.3, the native kernel is '3.19' -FROM ~~roct_container~~ +FROM ${roct_name} MAINTAINER Kent Knox # Global environment variables @@ -30,22 +30,27 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ rm -rf /var/lib/apt/lists/* # App specific environment variables -ENV ROCR_INSTALL_PATH=/opt/hsa ROCR_BUILD_PATH=/usr/local/src/ROCR-Runtime/ +ENV ROCR_INSTALL_PATH=${rocr_volume} ROCR_BUILD_PATH=/usr/local/src/ROCR-Runtime/ -# Configure, build and install +# Configure, build and install the runtime +# HSAIL based tools need the finalizer libraries, provided in the included deb file +# Extract only the finalizer libraries and build open source libhsa-runtime64 component on top RUN mkdir -p ${ROCR_BUILD_PATH} && \ cd ${ROCR_BUILD_PATH} && \ - git clone --depth=1 --branch=~~branch~~ https://github.com/RadeonOpenCompute/ROCR-Runtime.git ./ && \ + git clone --depth=1 --branch=${repo_branch} https://github.com/RadeonOpenCompute/ROCR-Runtime.git ./ && \ + dpkg --fsys-tarfile packages/ubuntu/*.deb | tar -xf - -C / --wildcards *.so* --exclude=libhsa-runtime64* && \ mkdir -p build && \ cd build && \ HSATHK_BUILD_INC_PATH=${HSATHK_INSTALL_PATH}/include \ HSATHK_BUILD_LIB_PATH=${HSATHK_INSTALL_PATH}/lib/x86_64-linux-gnu \ cmake \ -DCMAKE_INSTALL_PREFIX=${ROCR_INSTALL_PATH} \ - -DCMAKE_BUILD_TYPE=~~config~~ \ + -DCMAKE_BUILD_TYPE=${build_config} \ ../src && \ make -j $(nproc) install && \ cp -r ../src/inc ${ROCR_INSTALL_PATH}/include && \ - cd ~ && rm -rf ${ROCR_BUILD_PATH} && \ - echo "${ROCR_INSTALL_PATH}/lib" >> /etc/ld.so.conf.d/rocr.conf && \ + ${rocr_cleanup} \ + echo "${ROCR_INSTALL_PATH}lib" >> /etc/ld.so.conf.d/rocr.conf && \ ldconfig + +COPY setenv-rocr.sh ${ROCR_INSTALL_PATH} diff --git a/rocr/setenv-rocr.sh b/rocr/setenv-rocr.sh new file mode 100644 index 0000000..0f484bf --- /dev/null +++ b/rocr/setenv-rocr.sh @@ -0,0 +1,5 @@ +ROCR_PATH=/opt/hsa + +echo "Registering hsa libraries with loader" +echo "${ROCR_PATH}/lib" >> /etc/ld.so.conf.d/rocr.conf +ldconfig diff --git a/roct/roct-thunk-dockerfile.template b/roct/roct-thunk-dockerfile.template index 13907c7..dd08280 100644 --- a/roct/roct-thunk-dockerfile.template +++ b/roct/roct-thunk-dockerfile.template @@ -6,7 +6,7 @@ # This builds the radeon open compute kernel thunk # ubuntu:14.04.3, the native kernel is '3.19' -FROM ~~rock_container~~ +FROM ubuntu:14.04 MAINTAINER Kent Knox # Global environment variables @@ -27,15 +27,16 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ rm -rf /var/lib/apt/lists/* # App specific environment variables -ENV HSATHK_INSTALL_PATH=/usr HSATHK_BUILD_PATH=/usr/local/src/ROCT-Thunk-Interface +ENV HSATHK_INSTALL_PATH=${roct_volume} HSATHK_BUILD_PATH=/usr/local/src/ROCT-Thunk-Interface + +COPY setenv-roct.sh ${HSATHK_INSTALL_PATH} RUN mkdir -p ${HSATHK_BUILD_PATH} && \ cd ${HSATHK_BUILD_PATH} && \ - git clone --depth=1 --branch=~~branch~~ https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface.git ./ && \ - make -j $(nproc) ~~config~~ all deb && \ - rm -f ${HSATHK_INSTALL_PATH}/libhsakmt.* && \ - cp build/lnx64a/lib*.so.1 ${HSATHK_INSTALL_PATH}/lib/x86_64-linux-gnu && \ - cp -r include/ ${HSATHK_INSTALL_PATH} && \ - ln -s ${HSATHK_INSTALL_PATH}/lib/x86_64-linux-gnu/libhsakmt.so.1 ${HSATHK_INSTALL_PATH}/lib/x86_64-linux-gnu/libhsakmt.so && \ - cd ~ && rm -rf ${HSATHK_BUILD_PATH} && \ - ldconfig + git clone --depth=1 --branch=${repo_branch} https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface.git ./ && \ + make -j $(nproc) ${build_config_roct} all deb && \ + mkdir -p ${HSATHK_INSTALL_PATH}/lib/x86_64-linux-gnu && \ + cp -a include/ ${HSATHK_INSTALL_PATH} && \ + cp -a build/lnx64a/lib*.so* ${HSATHK_INSTALL_PATH}/lib/x86_64-linux-gnu && \ + ${roct_cleanup} \ + ${HSATHK_INSTALL_PATH}/setenv-roct.sh diff --git a/roct/setenv-roct.sh b/roct/setenv-roct.sh new file mode 100755 index 0000000..3d6679f --- /dev/null +++ b/roct/setenv-roct.sh @@ -0,0 +1,6 @@ +HSATHK_INSTALL_PATH=/opt/roct/lib/x86_64-linux-gnu +HSATHK_SYSTEM_PATH=/usr/lib/x86_64-linux-gnu + +echo "Copying hsa thunk into system directory" +rm -f ${HSATHK_SYSTEM_PATH}/libhsakmt.* +cp -a ${HSATHK_INSTALL_PATH}/lib*.so* ${HSATHK_SYSTEM_PATH}