Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
76fc63711a | ||
|
|
a49d0ba12c | ||
|
|
ae9b8518c3 | ||
|
|
28ec3f3b50 | ||
|
|
b115ad8036 |
+183
@@ -0,0 +1,183 @@
|
||||
[submodule "libs/ROCK-Kernel-Driver"]
|
||||
path = libs/ROCK-Kernel-Driver
|
||||
url = ../ROCK-Kernel-Driver
|
||||
[submodule "libs/ROCT-Thunk-Interface"]
|
||||
path = libs/ROCT-Thunk-Interface
|
||||
url = ../ROCT-Thunk-Interface
|
||||
[submodule "libs/ROCR-Runtime"]
|
||||
path = libs/ROCR-Runtime
|
||||
url = ../ROCR-Runtime
|
||||
[submodule "libs/amdsmi"]
|
||||
path = libs/amdsmi
|
||||
url = ../amdsmi
|
||||
[submodule "libs/rocm_smi_lib"]
|
||||
path = libs/rocm_smi_lib
|
||||
url = ../rocm_smi_lib
|
||||
[submodule "libs/rocm-core"]
|
||||
path = libs/rocm-core
|
||||
url = ../rocm-core
|
||||
[submodule "libs/rocm-cmake"]
|
||||
path = libs/rocm-cmake
|
||||
url = ../rocm-cmake
|
||||
[submodule "libs/rocminfo"]
|
||||
path = libs/rocminfo
|
||||
url = ../rocminfo
|
||||
[submodule "libs/rocm_bandwidth_test"]
|
||||
path = libs/rocm_bandwidth_test
|
||||
url = ../rocm_bandwidth_test
|
||||
[submodule "libs/rocprofiler"]
|
||||
path = libs/rocprofiler
|
||||
url = ../rocprofiler
|
||||
[submodule "libs/roctracer"]
|
||||
path = libs/roctracer
|
||||
url = ../roctracer
|
||||
[submodule "libs/rdc"]
|
||||
path = libs/rdc
|
||||
url = ../rdc
|
||||
[submodule "libs/HIP"]
|
||||
path = libs/HIP
|
||||
url = ../HIP
|
||||
[submodule "libs/clr"]
|
||||
path = libs/clr
|
||||
url = ../clr
|
||||
[submodule "libs/hipother"]
|
||||
path = libs/hipother
|
||||
url = ../hipother
|
||||
[submodule "libs/HIPIFY"]
|
||||
path = libs/HIPIFY
|
||||
url = ../HIPIFY
|
||||
[submodule "libs/llvm-project"]
|
||||
path = libs/llvm-project
|
||||
url = ../llvm-project
|
||||
[submodule "libs/half"]
|
||||
path = libs/half
|
||||
url = ../half
|
||||
[submodule "libs/ROCgdb"]
|
||||
path = libs/ROCgdb
|
||||
url = ../ROCgdb
|
||||
[submodule "libs/ROCdbgapi"]
|
||||
path = libs/ROCdbgapi
|
||||
url = ../ROCdbgapi
|
||||
[submodule "libs/rocr_debug_agent"]
|
||||
path = libs/rocr_debug_agent
|
||||
url = ../rocr_debug_agent
|
||||
[submodule "libs/rocBLAS"]
|
||||
path = libs/rocBLAS
|
||||
url = ../rocBLAS
|
||||
[submodule "libs/Tensile"]
|
||||
path = libs/Tensile
|
||||
url = ../Tensile
|
||||
[submodule "libs/hipTensor"]
|
||||
path = libs/hipTensor
|
||||
url = ../hipTensor
|
||||
[submodule "libs/hipBLAS"]
|
||||
path = libs/hipBLAS
|
||||
url = ../hipBLAS
|
||||
[submodule "libs/hipBLASLt"]
|
||||
path = libs/hipBLASLt
|
||||
url = ../hipBLASLt
|
||||
[submodule "libs/rocFFT"]
|
||||
path = libs/rocFFT
|
||||
url = ../rocFFT
|
||||
[submodule "libs/hipFFT"]
|
||||
path = libs/hipFFT
|
||||
url = ../hipFFT
|
||||
[submodule "libs/rocRAND"]
|
||||
path = libs/rocRAND
|
||||
url = ../rocRAND
|
||||
[submodule "libs/hipRAND"]
|
||||
path = libs/hipRAND
|
||||
url = ../hipRAND
|
||||
[submodule "libs/rocSPARSE"]
|
||||
path = libs/rocSPARSE
|
||||
url = ../rocSPARSE
|
||||
[submodule "libs/hipSPARSELt"]
|
||||
path = libs/hipSPARSELt
|
||||
url = ../hipSPARSELt
|
||||
[submodule "libs/rocSOLVER"]
|
||||
path = libs/rocSOLVER
|
||||
url = ../rocSOLVER
|
||||
[submodule "libs/hipSOLVER"]
|
||||
path = libs/hipSOLVER
|
||||
url = ../hipSOLVER
|
||||
[submodule "libs/hipSPARSE"]
|
||||
path = libs/hipSPARSE
|
||||
url = ../hipSPARSE
|
||||
[submodule "libs/rocALUTION"]
|
||||
path = libs/rocALUTION
|
||||
url = ../rocALUTION
|
||||
[submodule "libs/rocThrust"]
|
||||
path = libs/rocThrust
|
||||
url = ../rocThrust
|
||||
[submodule "libs/hipCUB"]
|
||||
path = libs/hipCUB
|
||||
url = ../hipCUB
|
||||
[submodule "libs/rocPRIM"]
|
||||
path = libs/rocPRIM
|
||||
url = ../rocPRIM
|
||||
[submodule "libs/rocWMMA"]
|
||||
path = libs/rocWMMA
|
||||
url = ../rocWMMA
|
||||
[submodule "libs/rccl"]
|
||||
path = libs/rccl
|
||||
url = ../rccl
|
||||
[submodule "libs/MIOpen"]
|
||||
path = libs/MIOpen
|
||||
url = ../MIOpen
|
||||
[submodule "libs/composable_kernel"]
|
||||
path = libs/composable_kernel
|
||||
url = ../composable_kernel
|
||||
[submodule "libs/MIVisionX"]
|
||||
path = libs/MIVisionX
|
||||
url = ../MIVisionX
|
||||
[submodule "libs/rpp"]
|
||||
path = libs/rpp
|
||||
url = ../rpp
|
||||
[submodule "libs/hipfort"]
|
||||
path = libs/hipfort
|
||||
url = ../hipfort
|
||||
[submodule "libs/AMDMIGraphX"]
|
||||
path = libs/AMDMIGraphX
|
||||
url = ../AMDMIGraphX
|
||||
[submodule "libs/ROCmValidationSuite"]
|
||||
path = libs/ROCmValidationSuite
|
||||
url = ../ROCmValidationSuite
|
||||
[submodule "libs/openmp-extras/aomp"]
|
||||
path = libs/openmp-extras/aomp
|
||||
url = ../aomp
|
||||
[submodule "libs/openmp-extras/aomp-extras"]
|
||||
path = libs/openmp-extras/aomp-extras
|
||||
url = ../aomp-extras
|
||||
[submodule "libs/openmp-extras/flang"]
|
||||
path = libs/openmp-extras/flang
|
||||
url = ../flang
|
||||
[submodule "libs/rocDecode"]
|
||||
path = libs/rocDecode
|
||||
url = ../rocDecode
|
||||
[submodule "libs/rocprofiler-sdk"]
|
||||
path = libs/rocprofiler-sdk
|
||||
url = ../rocprofiler-sdk
|
||||
[submodule "libs/rocm-examples"]
|
||||
path = libs/rocm-examples
|
||||
url = ../rocm-examples
|
||||
[submodule "libs/rocPyDecode"]
|
||||
path = libs/rocPyDecode
|
||||
url = ../rocPyDecode
|
||||
[submodule "libs/rocAL"]
|
||||
path = libs/rocAL
|
||||
url = ../rocAL
|
||||
[submodule "libs/rocprofiler-register"]
|
||||
path = libs/rocprofiler-register
|
||||
url = ../rocprofiler-register
|
||||
[submodule "libs/rocprofiler-compute"]
|
||||
path = libs/rocprofiler-compute
|
||||
url = ../rocprofiler-compute
|
||||
[submodule "libs/rocprofiler-systems"]
|
||||
path = libs/rocprofiler-systems
|
||||
url = ../rocprofiler-systems
|
||||
[submodule "libs/hip-tests"]
|
||||
path = libs/hip-tests
|
||||
url = ../hip-tests
|
||||
[submodule "libs/rocJPEG"]
|
||||
path = libs/rocJPEG
|
||||
url = ../rocJPEG
|
||||
@@ -21,19 +21,7 @@ source software compilers, debuggers, and libraries. ROCm is fully integrated in
|
||||
|
||||
## Getting the ROCm Source Code
|
||||
|
||||
AMD ROCm is built from open source software. It is, therefore, possible to modify the various components of ROCm by downloading the source code and rebuilding the components. The source code for ROCm components can be cloned from each of the GitHub repositories using git. For easy access to download the correct versions of each of these tools, the ROCm repository contains a repo manifest file called [default.xml](./default.xml). You can use this manifest file to download the source code for ROCm software.
|
||||
|
||||
### Installing the repo tool
|
||||
|
||||
The repo tool from Google allows you to manage multiple git repositories simultaneously. Run the following commands to install the repo tool:
|
||||
|
||||
```bash
|
||||
mkdir -p ~/bin/
|
||||
curl https://storage.googleapis.com/git-repo-downloads/repo > ~/bin/repo
|
||||
chmod a+x ~/bin/repo
|
||||
```
|
||||
|
||||
**Note:** The ```~/bin/``` folder is used as an example. You can specify a different folder to install the repo tool into if you desire.
|
||||
AMD ROCm is built from open source software. It is, therefore, possible to modify the various components of ROCm by downloading the source code and rebuilding the components. The source code for ROCm components can be cloned from each of the GitHub repositories using git. For easy access to download the correct versions of each of these tools, the ROCm repository contains submodules that point to the correct versions of each of the ROCm components. They can be found in the `/libs` directory of the ROCm repository.
|
||||
|
||||
### Installing git-lfs
|
||||
|
||||
@@ -45,17 +33,12 @@ sudo apt-get install git-lfs
|
||||
|
||||
### Downloading the ROCm source code
|
||||
|
||||
The following example shows how to use the repo tool to download the ROCm source code. If you choose a directory other than ~/bin/ to install the repo tool, you must use that chosen directory in the code as shown below:
|
||||
The following example shows how to download the ROCm source from this repository.
|
||||
|
||||
```bash
|
||||
mkdir -p ~/ROCm/
|
||||
cd ~/ROCm/
|
||||
~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.3.x
|
||||
~/bin/repo sync
|
||||
git clone https://github.com/ROCm/ROCm -b submodules-6-3-0 --recurse-submodules
|
||||
```
|
||||
|
||||
**Note:** Using this sample code will cause the repo tool to download the open source code associated with the specified ROCm release. Ensure that you have ssh-keys configured on your machine for your GitHub ID prior to the download as explained at [Connecting to GitHub with SSH](https://docs.github.com/en/authentication/connecting-to-github-with-ssh).
|
||||
|
||||
## Building the ROCm source code
|
||||
|
||||
Each ROCm component repository contains directions for building that component, such as the rocSPARSE documentation [Installation and Building for Linux](https://rocm.docs.amd.com/projects/rocSPARSE/en/latest/install/Linux_Install_Guide.html). Refer to the specific component documentation for instructions on building the repository.
|
||||
@@ -76,9 +59,8 @@ The Build time will reduce significantly if we limit the GPU Architecture/s agai
|
||||
|
||||
mkdir -p ~/WORKSPACE/ # Or any folder name other than WORKSPACE
|
||||
cd ~/WORKSPACE/
|
||||
export ROCM_VERSION=6.3.0
|
||||
~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.3.x -m tools/rocm-build/rocm-${ROCM_VERSION}.xml
|
||||
~/bin/repo sync
|
||||
export ROCM_VERSION=6.3.0 # or 6.2.0 6.1.0
|
||||
git clone https://github.com/ROCm/ROCm -b submodules-${ROCM_VERSION} --recurse-submodules
|
||||
|
||||
# --------------------------------------
|
||||
# Step 2: Prepare build environment
|
||||
@@ -167,12 +149,6 @@ Note: [Overview for ROCm.mk](tools/rocm-build/README.md)
|
||||
|
||||
## ROCm documentation
|
||||
|
||||
This repository contains the [manifest file](https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md)
|
||||
for ROCm releases, changelogs, and release information.
|
||||
|
||||
The `default.xml` file contains information for all repositories and the associated commit used to build
|
||||
the current ROCm release; `default.xml` uses the [Manifest Format repository](https://gerrit.googlesource.com/git-repo/).
|
||||
|
||||
Source code for our documentation is located in the `/docs` folder of most ROCm repositories. The
|
||||
`develop` branch of our repositories contains content for the next ROCm release.
|
||||
|
||||
|
||||
+3
-2
@@ -10,8 +10,6 @@
|
||||
<project name="ROCR-Runtime" />
|
||||
<project name="ROCT-Thunk-Interface" />
|
||||
<project name="amdsmi" />
|
||||
<project name="omniperf" />
|
||||
<project name="omnitrace" />
|
||||
<project name="rdc" />
|
||||
<project name="rocm_bandwidth_test" />
|
||||
<project name="rocm_smi_lib" />
|
||||
@@ -21,6 +19,8 @@
|
||||
<project name="rocprofiler" />
|
||||
<project name="rocprofiler-register" />
|
||||
<project name="rocprofiler-sdk" />
|
||||
<project name="rocprofiler-compute" />
|
||||
<project name="rocprofiler-systems" />
|
||||
<project name="roctracer" />
|
||||
<!--HIP Projects-->
|
||||
<project name="HIP" />
|
||||
@@ -57,6 +57,7 @@
|
||||
<project groups="mathlibs" name="rocALUTION" />
|
||||
<project groups="mathlibs" name="rocBLAS" />
|
||||
<project groups="mathlibs" name="rocDecode" />
|
||||
<project groups="mathlibs" name="rocJPEG" />
|
||||
<project groups="mathlibs" name="rocPyDecode" />
|
||||
<project groups="mathlibs" name="rocFFT" />
|
||||
<project groups="mathlibs" name="rocPRIM" />
|
||||
|
||||
Submodule
+1
Submodule libs/AMDMIGraphX added at 4b20cbc960
Submodule
+1
Submodule libs/HIP added at 2451439986
Submodule
+1
Submodule libs/HIPIFY added at f156b776ec
Submodule
+1
Submodule libs/MIOpen added at a85ca8a54f
Submodule
+1
Submodule libs/MIVisionX added at f0e86f9638
Submodule
+1
Submodule libs/ROCK-Kernel-Driver added at 627f8902a9
Submodule
+1
Submodule libs/ROCR-Runtime added at 8135ac22b4
Submodule
+1
Submodule libs/ROCT-Thunk-Interface added at c8db22210d
Submodule
+1
Submodule libs/ROCdbgapi added at 61dfa5620c
Submodule
+1
Submodule libs/ROCgdb added at 16782e8d04
Submodule
+1
Submodule libs/ROCmValidationSuite added at fdb44459ec
Submodule
+1
Submodule libs/Tensile added at df4be50f2c
Submodule
+1
Submodule libs/amdsmi added at 2858e51b4e
Submodule
+1
Submodule libs/clr added at fa1d09cbda
Submodule
+1
Submodule libs/composable_kernel added at a9894ecbe3
Submodule
+1
Submodule libs/half added at 1ddada2251
Submodule
+1
Submodule libs/hip-tests added at 61f4742c4d
Submodule
+1
Submodule libs/hipBLAS added at e75831fa5e
Submodule
+1
Submodule libs/hipBLASLt added at 9f30df5504
Submodule
+1
Submodule libs/hipCUB added at a4b3b193ee
Submodule
+1
Submodule libs/hipFFT added at 85a2d27308
Submodule
+1
Submodule libs/hipRAND added at 08810454ad
Submodule
+1
Submodule libs/hipSOLVER added at be093932fd
Submodule
+1
Submodule libs/hipSPARSE added at 54071d80f5
Submodule
+1
Submodule libs/hipSPARSELt added at ac2f73ab21
Submodule
+1
Submodule libs/hipTensor added at 68014b5c76
Submodule
+1
Submodule libs/hipfort added at e068c4372e
Submodule
+1
Submodule libs/hipother added at ed5b3fbed8
Submodule
+1
Submodule libs/llvm-project added at f24aa3b4a9
Submodule
+1
Submodule libs/openmp-extras/aomp added at 48e22df4f2
Submodule
+1
Submodule libs/openmp-extras/aomp-extras added at abc77799fa
Submodule
+1
Submodule libs/openmp-extras/flang added at 64722961f1
Submodule
+1
Submodule libs/rccl added at eef7b2918c
Submodule
+1
Submodule libs/rdc added at 3bb5c700a1
Submodule
+1
Submodule libs/rocAL added at 3947ad031c
Submodule
+1
Submodule libs/rocALUTION added at 9ed79898ae
Submodule
+1
Submodule libs/rocBLAS added at 8ebd6c11c4
Submodule
+1
Submodule libs/rocDecode added at a99eb1d3d7
Submodule
+1
Submodule libs/rocFFT added at 3806d68ba7
Submodule
+1
Submodule libs/rocJPEG added at d2e513133c
Submodule
+1
Submodule libs/rocPRIM added at 1eefdb730b
Submodule
+1
Submodule libs/rocPyDecode added at 2025c51b0d
Submodule
+1
Submodule libs/rocRAND added at 07b4fba423
Submodule
+1
Submodule libs/rocSOLVER added at ee3ec9d2c2
Submodule
+1
Submodule libs/rocSPARSE added at ed5da19958
Submodule
+1
Submodule libs/rocThrust added at c33822fe39
Submodule
+1
Submodule libs/rocWMMA added at ba38cf3015
Submodule
+1
Submodule libs/rocm-cmake added at 489ef56a39
Submodule
+1
Submodule libs/rocm-core added at ce2be3b65a
Submodule
+1
Submodule libs/rocm-examples added at 62b67a2af5
Submodule
+1
Submodule libs/rocm_bandwidth_test added at eb36f41e7f
Submodule
+1
Submodule libs/rocm_smi_lib added at 55c0f58588
Submodule
+1
Submodule libs/rocminfo added at 9f6d7cdf6b
Submodule
+1
Submodule libs/rocprofiler added at ec0e166dfe
Submodule
+1
Submodule libs/rocprofiler-compute added at 4244ddb079
Submodule
+1
Submodule libs/rocprofiler-register added at b71f9cabe6
Submodule
+1
Submodule libs/rocprofiler-sdk added at 38ac1c8f7d
Submodule
+1
Submodule libs/rocprofiler-systems added at 71a5e271b5
Submodule
+1
Submodule libs/rocr_debug_agent added at a1df2070eb
Submodule
+1
Submodule libs/roctracer added at d5bbc8aa77
Submodule
+1
Submodule libs/rpp added at fe68edd53a
@@ -0,0 +1,331 @@
|
||||
|
||||
'''
|
||||
This script sets up git submodules based on a manifest file. It performs the following tasks:
|
||||
1. Finds the root directory of the project.
|
||||
2. Parses command line arguments to get the manifest file and output folder paths.
|
||||
3. Parses the manifest file to get a list of projects.
|
||||
4. Gets a list of existing submodules.
|
||||
5. Removes submodules that are not in the manifest file.
|
||||
6. Adds new submodules from the manifest file.
|
||||
7. Updates existing submodules to the specified revision.
|
||||
'''
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import argparse
|
||||
import pathlib
|
||||
import logging
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
class Project:
|
||||
"""
|
||||
Project class to store project information.
|
||||
|
||||
Attributes:
|
||||
name (str): Name of the project.
|
||||
revision (str): Revision of the project.
|
||||
path (str): Path to the project.
|
||||
|
||||
Methods:
|
||||
__str__: Returns a string representation of the project.
|
||||
__repr__: Returns a string representation of the project.
|
||||
"""
|
||||
def __init__(self, **kwargs):
|
||||
'''
|
||||
Project class to store project information
|
||||
:param name: Name of the project
|
||||
:param revision: Revision of the project
|
||||
:param path: Path to the project
|
||||
'''
|
||||
self.name = kwargs.get('name')
|
||||
self.revision = kwargs.get('revision')
|
||||
self.path = kwargs.get('path')
|
||||
|
||||
|
||||
def __str__(self):
|
||||
return f"Project {self.name} at {self.path} with revision {self.revision}"
|
||||
|
||||
|
||||
def __repr__(self):
|
||||
return self.__str__()
|
||||
|
||||
|
||||
# Color codes
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
BLUE='\033[0;34m'
|
||||
DARK_RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
|
||||
def find_root() -> str:
|
||||
'''
|
||||
Find the root directory of the project
|
||||
:return: Root directory of the project
|
||||
'''
|
||||
current = os.getcwd()
|
||||
while current != "/":
|
||||
if os.path.exists(os.path.join(current, ".git")):
|
||||
return current
|
||||
current = os.path.dirname(current)
|
||||
print("Could not find root directory", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def run_command(command) -> list:
|
||||
'''
|
||||
Run a command in the shell
|
||||
:param command: Command to run
|
||||
:return: List of errors
|
||||
'''
|
||||
error_log = []
|
||||
print(f"Running command: {command}")
|
||||
# subprocess.run(command, shell=True, check=True)
|
||||
result = subprocess.run(command, shell=True, check=False,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f"{RED}Error running command: {result.stderr.decode('utf-8')}{NC}",
|
||||
file=sys.stderr)
|
||||
|
||||
logging.error("Error running command: %s", command)
|
||||
logging.error("\t%s", result.stderr.decode('utf-8'))
|
||||
|
||||
error_log.append(f"Error running command: {command}")
|
||||
error_log.append(f"\t{result.stderr.decode('utf-8')}\n")
|
||||
else:
|
||||
if result.stdout:
|
||||
print(result.stdout.decode('utf-8'))
|
||||
|
||||
return error_log
|
||||
|
||||
|
||||
def parse_arguments(project_root: str) -> argparse.Namespace:
|
||||
'''
|
||||
Parse command line arguments
|
||||
:param project_root: Root directory of the project
|
||||
:return: Parsed arguments
|
||||
'''
|
||||
|
||||
parser = argparse.ArgumentParser(description="Setup submodules based on a manifest file.")
|
||||
parser.add_argument("-m", "--manifest", required=False,
|
||||
default=f'{project_root}/default.xml', help="Path to the manifest file")
|
||||
parser.add_argument("-o", "--output", required=False,
|
||||
default='libs', help="Path to the submodule folder")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def parse_manifest(manifest_path: str, output_path: str) -> list:
|
||||
'''
|
||||
Parse the manifest file and return a list of projects
|
||||
:param manifest_path: Path to the manifest file
|
||||
:param output_path: Path to the output folder
|
||||
:return: List of projects
|
||||
'''
|
||||
|
||||
tree = ET.parse(manifest_path)
|
||||
root = tree.getroot()
|
||||
|
||||
defaults = root.find('default')
|
||||
default_revision = None
|
||||
|
||||
if defaults is not None:
|
||||
default_revision = defaults.get('revision')
|
||||
# print(f"Default revision: {default_revision}")
|
||||
|
||||
projects = []
|
||||
for project in root.findall('project'):
|
||||
name = project.get('name')
|
||||
revision = project.get('revision', default_revision)
|
||||
path = os.path.join(output_path, project.get('path', name))
|
||||
|
||||
projects.append(Project(name = name,
|
||||
revision = revision,
|
||||
path = path))
|
||||
|
||||
return projects
|
||||
|
||||
|
||||
def get_existing_submodules(project_root: str) -> list:
|
||||
'''
|
||||
Get a list of existing submodules
|
||||
:param project_root: Root directory of the project
|
||||
:return: List of existing submodules
|
||||
'''
|
||||
|
||||
gitmodules_path = os.path.join(project_root, ".gitmodules")
|
||||
existing_submodules = []
|
||||
|
||||
if os.path.exists(gitmodules_path):
|
||||
with open(gitmodules_path, "r", encoding="utf-8") as gitmodules_file:
|
||||
|
||||
for line in gitmodules_file:
|
||||
line = line.strip()
|
||||
|
||||
if line.startswith("[submodule"):
|
||||
submodule_name = pathlib.Path(line.split('"')[1]).as_posix()
|
||||
existing_submodules.append(submodule_name)
|
||||
|
||||
return existing_submodules
|
||||
|
||||
|
||||
def remove_submodule(project_root: str, module_path: str) -> list:
|
||||
'''
|
||||
Remove a submodule
|
||||
:param project_root: Root directory of the project
|
||||
:param module_path: Path to the submodule
|
||||
:return: List of errors
|
||||
'''
|
||||
error_log = []
|
||||
gitmodules_path = os.path.join(project_root, ".gitmodules")
|
||||
|
||||
error_log.extend(run_command(f"git submodule deinit -f {module_path}"))
|
||||
error_log.extend(run_command(f"rm -rf {module_path}"))
|
||||
error_log.extend(run_command(f"rm -rf .git/modules/{module_path}"))
|
||||
# error_log.extend(run_command(f"git rm -r --cached {module_path}"))
|
||||
|
||||
# # Remove the submodule from the .gitmodules file
|
||||
with open(gitmodules_path, "r", encoding="utf-8") as gitmodules_file:
|
||||
lines = gitmodules_file.readlines()
|
||||
|
||||
with open(gitmodules_path, "w", encoding="utf-8") as gitmodules_file:
|
||||
skip = False
|
||||
for line in lines:
|
||||
if line.strip().startswith(f"[submodule \"{module_path}\"]"):
|
||||
skip = True
|
||||
elif skip and line.strip().startswith("[submodule"):
|
||||
skip = False
|
||||
|
||||
if not skip:
|
||||
gitmodules_file.write(line)
|
||||
|
||||
return error_log
|
||||
|
||||
|
||||
def main():
|
||||
'''
|
||||
Main function
|
||||
'''
|
||||
|
||||
# Set up logging
|
||||
log_file = os.path.join(os.path.expanduser("~"), "submodule_setup.log")
|
||||
logging.basicConfig(level=logging.DEBUG,
|
||||
filename=log_file,
|
||||
filemode='w',
|
||||
format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
|
||||
# Save errors to a list to output at the end
|
||||
error_log = []
|
||||
|
||||
project_root = find_root()
|
||||
os.chdir(project_root)
|
||||
|
||||
print(f"Found project root at {project_root}")
|
||||
|
||||
args = parse_arguments(project_root)
|
||||
print(f"Using manifest file at {args.manifest}")
|
||||
|
||||
# Check if manifest file exists
|
||||
if not os.path.exists(args.manifest):
|
||||
print(f"Manifest file {args.manifest} does not exist", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Check if output folder exists
|
||||
print(f"Using output folder at {args.output}\n")
|
||||
if not os.path.exists(args.output):
|
||||
print(f"Output folder {args.output} does not exist. Creating it.")
|
||||
os.makedirs(args.output)
|
||||
|
||||
# Create a list of existing submodules
|
||||
existing_submodules = get_existing_submodules(project_root)
|
||||
|
||||
for module in existing_submodules:
|
||||
logging.debug("Existing submodule %s", module)
|
||||
|
||||
logging.debug("=====\n")
|
||||
|
||||
# Create a list of projects from the manifest file
|
||||
projects = parse_manifest(args.manifest, args.output)
|
||||
|
||||
for project in projects:
|
||||
logging.debug("Manifest Project %s to %s at %s",
|
||||
project.name,
|
||||
project.path,
|
||||
project.revision)
|
||||
|
||||
logging.debug("=====\n")
|
||||
|
||||
print('\n*** Initializing submodules ***\n')
|
||||
run_command("git submodule update --init")
|
||||
|
||||
print(f'\n{DARK_RED}*** Removing old projects ***{NC}\n')
|
||||
|
||||
# If project is in existing submodules and not in manifest, remove it
|
||||
removed_modules = []
|
||||
|
||||
project_paths = {project.path for project in projects}
|
||||
for module in existing_submodules:
|
||||
if module not in project_paths:
|
||||
print(f"{DARK_RED}Removing submodule {module}{NC}")
|
||||
logging.info("Removing submodule %s", module)
|
||||
|
||||
removed_modules.append(module)
|
||||
error_log.extend(remove_submodule(project_root, module))
|
||||
|
||||
print()
|
||||
|
||||
# Remove the submodules from the list of existing submodules
|
||||
existing_submodules = [module for module in existing_submodules
|
||||
if module not in removed_modules]
|
||||
|
||||
print(f'\n{GREEN}*** Adding new projects ***{NC}\n')
|
||||
|
||||
# Add new submodules
|
||||
for project in projects:
|
||||
if project.path not in existing_submodules:
|
||||
print(f"{GREEN}Adding submodule ../{project.name}{NC}")
|
||||
logging.info("Adding submodule ../%s to %s", project.name, project.path)
|
||||
|
||||
error_log.extend(
|
||||
run_command(f"git submodule add ../{project.name} {project.path}"))
|
||||
error_log.extend(
|
||||
run_command(f"cd {project.path} "
|
||||
f"&& git checkout {project.revision} "
|
||||
f"&& cd {project_root}"))
|
||||
print()
|
||||
|
||||
print(f'\n{BLUE}*** Updating existing projects ***{NC}\n')
|
||||
|
||||
# Update existing submodules
|
||||
for project in projects:
|
||||
if project.path in existing_submodules and project.path not in removed_modules:
|
||||
|
||||
print(f"{BLUE}Updating submodule {project.path}{NC}")
|
||||
logging.info("Updating submodule %s", project.path)
|
||||
|
||||
error_log.extend(
|
||||
run_command(f"cd {project.path} && "
|
||||
"git fetch --tag && "
|
||||
f"git checkout {project.revision} "
|
||||
f"&& cd {project_root}"))
|
||||
print()
|
||||
|
||||
# Print errors at the end
|
||||
if error_log:
|
||||
print(f"\n{RED}Errors occurred. Please check the logs: {log_file}{NC}\n"
|
||||
"Error Summary:\n",
|
||||
file=sys.stderr)
|
||||
logging.error("Errors occurred. Please check the logs: %s", log_file)
|
||||
logging.error("Error Summary:")
|
||||
|
||||
for error in error_log:
|
||||
print(error, file=sys.stderr)
|
||||
logging.error(error)
|
||||
else:
|
||||
print(f"{GREEN}All submodules updated successfully{NC}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+150
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/bash
|
||||
|
||||
# Color codes
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
|
||||
printHelp() {
|
||||
# Print the help message
|
||||
printf -- \
|
||||
"
|
||||
Usage: update-submodules.sh [-t <tag>] [-p <path>]
|
||||
Description: Update submodules to the ROCm release tag
|
||||
Options:
|
||||
-h, --help: Print the help message
|
||||
-t, --tag: The ROCm release tag to update the submodules to
|
||||
-p, --path: The path to the submodules directory
|
||||
|
||||
Example:
|
||||
./update-submodules.sh -t rocm-6.1.0 -p ../../libs"
|
||||
}
|
||||
|
||||
updateSubmodules() {
|
||||
# Update the submodules in the given directory to the desired tag
|
||||
# $1: The directory to update the submodules in
|
||||
|
||||
local directory=$1
|
||||
pushd "$directory" >> /dev/null || exit
|
||||
|
||||
for d in */
|
||||
do
|
||||
pushd "$d" >> /dev/null || exit
|
||||
|
||||
if [[ -e .git ]]; then
|
||||
echo -e "${GREEN}${d} is a git folder${NC}"
|
||||
|
||||
if ! git fetch --all; then
|
||||
echo -e "${RED}Failed to fetch for ${d}${NC}"
|
||||
|
||||
# Save the current directory to an array to output the list of
|
||||
# failed fetches at the end.
|
||||
gitFetchErrors+=("$d")
|
||||
fi
|
||||
|
||||
# Checkout the desired tag
|
||||
if ! git checkout tags/"${RocmRelease}"; then
|
||||
echo -e "${RED}Failed to checkout tag ${RocmRelease} for ${d}${NC}"
|
||||
|
||||
# Save the current directory to an array to output the list of
|
||||
# failed checkouts at the end.
|
||||
gitCheckoutErrors+=("$d")
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}${d} is NOT a git folder${NC}"
|
||||
|
||||
# Save the current directory to an array to output the list of
|
||||
# non-git folders at the end.
|
||||
nonGitFolders+=("$d")
|
||||
fi
|
||||
echo -e "${NC}"
|
||||
popd >> /dev/null || exit
|
||||
done
|
||||
popd >> /dev/null || exit
|
||||
}
|
||||
|
||||
#######
|
||||
# MAIN PROGRAM
|
||||
#######
|
||||
|
||||
# Default values
|
||||
RocmRelease=rocm-6.1.0
|
||||
PathToSubmodules=../../libs
|
||||
|
||||
# Parse command line parameters.
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
-h | --help )
|
||||
printHelp
|
||||
exit 0
|
||||
;;
|
||||
-t | --tag )
|
||||
shift
|
||||
RocmRelease=$1
|
||||
shift
|
||||
;;
|
||||
-p | --path )
|
||||
shift
|
||||
PathToSubmodules=$1
|
||||
shift
|
||||
;;
|
||||
* )
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "********************************"
|
||||
echo -e "${BLUE}Path to Submodules: ${PathToSubmodules}${NC}"
|
||||
pushd "$PathToSubmodules" >> /dev/null || exit
|
||||
|
||||
echo -e "${BLUE}Syncing to Tag: ${RocmRelease}.${NC}"
|
||||
echo "********************************"
|
||||
echo
|
||||
|
||||
# Update submodules in the current directory
|
||||
updateSubmodules "."
|
||||
|
||||
# Update the `openmp-extras` modules
|
||||
updateSubmodules "openmp-extras"
|
||||
|
||||
popd >> /dev/null || exit
|
||||
|
||||
# Output summary of errors
|
||||
|
||||
echo "********************************"
|
||||
echo -e "${BLUE}*** Error Report ***${NC}"
|
||||
echo "********************************"
|
||||
echo
|
||||
|
||||
if [ ${#nonGitFolders[@]} -gt 0 ]; then
|
||||
echo -e "${RED}The following folders are not git folders:${NC}"
|
||||
|
||||
for d in "${nonGitFolders[@]}"
|
||||
do
|
||||
echo "${d}"
|
||||
done
|
||||
echo
|
||||
fi
|
||||
|
||||
if [ ${#gitFetchErrors[@]} -gt 0 ]; then
|
||||
echo -e "${RED}The following folders failed to fetch:${NC}"
|
||||
|
||||
for d in "${gitFetchErrors[@]}"
|
||||
do
|
||||
echo "${d}"
|
||||
done
|
||||
echo
|
||||
fi
|
||||
|
||||
if [ ${#gitCheckoutErrors[@]} -gt 0 ]; then
|
||||
echo -e "${RED}The following folders failed to checkout tag ${RocmRelease}:${NC}"
|
||||
|
||||
for d in "${gitCheckoutErrors[@]}"
|
||||
do
|
||||
echo "${d}"
|
||||
done
|
||||
echo
|
||||
fi
|
||||
Reference in New Issue
Block a user