hanzo/logs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ci: enable helm-loki-ci with helm diff (#15734)

Browse Source
This commit is contained in:
Jackson Coelho
2025-01-15 14:29:54 +01:00
committed by GitHub
parent 11714a71c7
commit c9b584ebbb
4 changed files with 450 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/actions/setup-k3d/action.yaml
+22
View File
@@ -0,0 +1,22 @@
name: Setup K3D Cluster
description: Setup a K3D cluster
runs:
using: "composite"
steps:
- name: Install k3d
shell: bash
run: |
curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
- name: Create k3d cluster with local registry
shell: bash
run: |
k3d registry create registry.local --port 5000
k3d cluster create k3d-cluster --registry-use k3d-registry.local:5000
- name: Set kubectl context to k3d cluster
shell: bash
run: |
k3d kubeconfig merge k3d-cluster --kubeconfig-switch-context
.github/workflows/helm-diff-ci.yml
+374
View File
@@ -0,0 +1,374 @@
---
name: Helm Loki Diff CI
on:
pull_request:
paths:
- 'production/helm/loki/**'
permissions:
contents: read
pull-requests: write
jobs:
single-binary-diff:
name: Single Binary Scenario
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Helm
uses: azure/setup-helm@v4
- name: Add required Helm repositories
run: |
helm repo add minio https://charts.min.io/
helm repo add grafana https://grafana.github.io/helm-charts
helm repo add grafana-operator https://grafana.github.io/helm-charts
helm repo update
- name: Setup K3D
uses: ./.github/actions/setup-k3d
- name: Setup Helm plugins
run: |
helm plugin install https://github.com/databus23/helm-diff
- name: Build helm dependencies
run: |
helm dependency build production/helm/loki
- name: Install latest helm release
run: |
helm install --create-namespace loki-release grafana/loki -f production/helm/loki/scenarios/default-single-binary-values.yaml
- name: Run helm diff
id: helm-diff
env:
HELM_DIFF_USE_UPGRADE_DRY_RUN: true
run: |
helm diff upgrade loki-release -f production/helm/loki/scenarios/default-single-binary-values.yaml production/helm/loki | tee helm_diff_output.txt
- name: Convert Helm Diff Output to Markdown
id: convert_diff
run: |
cat helm_diff_output.txt >> formatted_diff_output.md
- name: Upload diff output as artifact
id: upload_diff
uses: actions/upload-artifact@v4
with:
name: single-binary-diff-output
path: formatted_diff_output.md
retention-days: 2
default-values-diff:
name: Default Values Scenario
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Helm
uses: azure/setup-helm@v4
- name: Add required Helm repositories
run: |
helm repo add minio https://charts.min.io/
helm repo add grafana https://grafana.github.io/helm-charts
helm repo add grafana-operator https://grafana.github.io/helm-charts
helm repo update
- name: Setup K3D
uses: ./.github/actions/setup-k3d
- name: Setup Helm plugins
run: |
helm plugin install https://github.com/databus23/helm-diff
- name: Build helm dependencies
run: |
helm dependency build production/helm/loki
- name: Install latest helm release
run: |
helm install --create-namespace loki-release grafana/loki -f production/helm/loki/scenarios/default-values.yaml
- name: Run helm diff
id: helm-diff
env:
HELM_DIFF_USE_UPGRADE_DRY_RUN: true
run: |
helm diff upgrade loki-release -f production/helm/loki/scenarios/default-values.yaml production/helm/loki | tee helm_diff_output.txt
- name: Convert Helm Diff Output to Markdown
id: convert_diff
run: |
cat helm_diff_output.txt >> formatted_diff_output.md
- name: Upload diff output as artifact
uses: actions/upload-artifact@v4
id: upload_diff
with:
name: default-values-diff-output
path: formatted_diff_output.md
retention-days: 2
ingress-values-diff:
name: Ingress Values Scenario
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Helm
uses: azure/setup-helm@v4
- name: Add required Helm repositories
run: |
helm repo add minio https://charts.min.io/
helm repo add grafana https://grafana.github.io/helm-charts
helm repo add grafana-operator https://grafana.github.io/helm-charts
helm repo update
- name: Setup K3D
uses: ./.github/actions/setup-k3d
- name: Setup Helm plugins
run: |
helm plugin install https://github.com/databus23/helm-diff
- name: Build helm dependencies
run: |
helm dependency build production/helm/loki
- name: Install latest helm release
run: |
helm install --create-namespace loki-release grafana/loki -f production/helm/loki/scenarios/ingress-values.yaml
- name: Run helm diff
id: helm-diff
env:
HELM_DIFF_USE_UPGRADE_DRY_RUN: true
run: |
helm diff upgrade loki-release -f production/helm/loki/scenarios/ingress-values.yaml production/helm/loki | tee helm_diff_output.txt
- name: Convert Helm Diff Output to Markdown
id: convert_diff
run: |
cat helm_diff_output.txt >> formatted_diff_output.md
- name: Upload diff output as artifact
uses: actions/upload-artifact@v4
id: upload_diff
with:
name: ingress-diff-output
path: formatted_diff_output.md
retention-days: 2
legacy-monitoring-values-diff:
name: Legacy Monitoring Values Scenario
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Helm
uses: azure/setup-helm@v4
- name: Add required Helm repositories
run: |
helm repo add minio https://charts.min.io/
helm repo add grafana https://grafana.github.io/helm-charts
helm repo add grafana-operator https://grafana.github.io/helm-charts
helm repo update
- name: Setup K3D
uses: ./.github/actions/setup-k3d
- name: Setup Helm plugins
run: |
helm plugin install https://github.com/databus23/helm-diff
- name: Build helm dependencies
run: |
helm dependency build production/helm/loki
- name: Install latest helm release
run: |
helm install --create-namespace loki-release grafana/loki -f production/helm/loki/scenarios/legacy-monitoring-values.yaml
- name: Run helm diff
id: helm-diff
env:
HELM_DIFF_USE_UPGRADE_DRY_RUN: true
run: |
helm diff upgrade loki-release -f production/helm/loki/scenarios/legacy-monitoring-values.yaml production/helm/loki | tee helm_diff_output.txt
- name: Convert Helm Diff Output to Markdown
id: convert_diff
run: |
cat helm_diff_output.txt >> formatted_diff_output.md
- name: Upload diff output as artifact
uses: actions/upload-artifact@v4
id: upload_diff
with:
name: legacy-monitoring-diff-output
path: formatted_diff_output.md
retention-days: 2
simple-scalable-aws-kube-irsa-values-diff:
name: Simple Scalable AWS Kube IRSA Values Scenario
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Helm
uses: azure/setup-helm@v4
- name: Add required Helm repositories
run: |
helm repo add minio https://charts.min.io/
helm repo add grafana https://grafana.github.io/helm-charts
helm repo add grafana-operator https://grafana.github.io/helm-charts
helm repo update
- name: Build helm dependencies
run: |
helm dependency build production/helm/loki
- name: Generate latest manifests
run: |
helm template loki-release grafana/loki -f production/helm/loki/scenarios/simple-scalable-aws-kube-irsa-values.yaml > release-manifest.yaml
- name: Generate current manifest
run: |
helm template loki-release production/helm/loki -f production/helm/loki/scenarios/simple-scalable-aws-kube-irsa-values.yaml > current-manifest.yaml
- name: Compare manifests
run: |
diff current-manifest.yaml release-manifest.yaml > helm_diff_output.txt || true
- name: Convert Helm Diff Output to Markdown
id: convert_diff
run: |
cat helm_diff_output.txt >> formatted_diff_output.md
- name: Upload diff output as artifact
uses: actions/upload-artifact@v4
id: upload_diff
with:
name: simple-scalable-aws-kube-irsa-diff-output
path: formatted_diff_output.md
retention-days: 2
summary-diff-outputs:
name: Summary Diffs
runs-on: ubuntu-latest
needs:
- single-binary-diff
- default-values-diff
- ingress-values-diff
- legacy-monitoring-values-diff
- simple-scalable-aws-kube-irsa-values-diff
steps:
- name: Checkout code
uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
name: single-binary-diff-output
path: single-binary-diff
- uses: actions/download-artifact@v4
with:
name: default-values-diff-output
path: default-values-diff
- uses: actions/download-artifact@v4
with:
name: ingress-diff-output
path: ingress-values-diff
- uses: actions/download-artifact@v4
with:
name: legacy-monitoring-diff-output
path: legacy-monitoring-values-diff
- uses: actions/download-artifact@v4
with:
name: simple-scalable-aws-kube-irsa-diff-output
path: simple-scalable-aws-kube-irsa-values-diff
- name: Combine diff outputs
run: |
echo "## Helm Diff Output - Summary" > formatted_diff_output.md
echo "<details>" >> formatted_diff_output.md
echo "" >> formatted_diff_output.md
echo "<summary>Single Binary Scenario</summary>" >> formatted_diff_output.md
echo "" >> formatted_diff_output.md
echo '```diff' >> formatted_diff_output.md
cat single-binary-diff/formatted_diff_output.md >> formatted_diff_output.md
echo '```' >> formatted_diff_output.md
echo "</details>" >> formatted_diff_output.md
echo "" >> formatted_diff_output.md
echo "<details>" >> formatted_diff_output.md
echo "" >> formatted_diff_output.md
echo "<summary>Default Values Scenario</summary>" >> formatted_diff_output.md
echo "" >> formatted_diff_output.md
echo '```diff' >> formatted_diff_output.md
cat default-values-diff/formatted_diff_output.md >> formatted_diff_output.md
echo '```' >> formatted_diff_output.md
echo "</details>" >> formatted_diff_output.md
echo "" >> formatted_diff_output.md
echo "<details>" >> formatted_diff_output.md
echo "" >> formatted_diff_output.md
echo "<summary>Ingress Values Scenario</summary>" >> formatted_diff_output.md
echo "" >> formatted_diff_output.md
echo '```diff' >> formatted_diff_output.md
cat ingress-values-diff/formatted_diff_output.md >> formatted_diff_output.md
echo '```' >> formatted_diff_output.md
echo "</details>" >> formatted_diff_output.md
echo "" >> formatted_diff_output.md
echo "<details>" >> formatted_diff_output.md
echo "" >> formatted_diff_output.md
echo "<summary>Legacy Monitoring Scenario</summary>" >> formatted_diff_output.md
echo "" >> formatted_diff_output.md
echo '```diff' >> formatted_diff_output.md
cat legacy-monitoring-values-diff/formatted_diff_output.md >> formatted_diff_output.md
echo '```' >> formatted_diff_output.md
echo "</details>" >> formatted_diff_output.md
echo "" >> formatted_diff_output.md
echo "<details>" >> formatted_diff_output.md
echo "" >> formatted_diff_output.md
echo "<summary>Simple Scalable AWS Kube IRSA Scenario</summary>" >> formatted_diff_output.md
echo "" >> formatted_diff_output.md
echo '```diff' >> formatted_diff_output.md
cat simple-scalable-aws-kube-irsa-values-diff/formatted_diff_output.md >> formatted_diff_output.md
echo '```' >> formatted_diff_output.md
echo "</details>" >> formatted_diff_output.md
echo "" >> formatted_diff_output.md
- name: Post diff as PR comment
uses: marocchino/sticky-pull-request-comment@v2
with:
header: "Helm Diff Output - Summary"
skip_unchanged: true
hide_and_recreate: true
append: true
hide_classify: "OUTDATED"
path: formatted_diff_output.md
.github/workflows/helm-loki-ci.yml
-221
View File
@@ -1,221 +0,0 @@
---
name: helm-loki-ci
on:
pull_request:
paths:
- "production/helm/loki/**"
jobs:
publish-diff:
# temporarily disable the workflow for the PRs where PRs branch is from fork.
if: github.event.pull_request.head.repo.full_name == github.repository
name: Publish Rendered Helm Chart Diff
runs-on: ubuntu-latest
steps:
- name: Setup Helm
uses: azure/setup-helm@v4
- name: Add required Helm repositories
run: |
helm repo add minio https://charts.min.io/
helm repo add grafana https://grafana.github.io/helm-charts
helm repo add grafana-operator https://grafana.github.io/helm-charts
helm repo update
- name: Prepare directories for base and PR branches
run: |
mkdir -p ${{ github.workspace }}/base
mkdir -p ${{ github.workspace }}/pr
mkdir -p ${{ github.workspace }}/output
mkdir -p ${{ github.workspace }}/output/base
mkdir -p ${{ github.workspace }}/output/pr
- name: Checkout base branch to 'base' folder within workspace
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.base.ref }}
repository: ${{ github.event.pull_request.base.repo.full_name }}
path: ${{ github.workspace }}/base
- name: Checkout PR branch to 'pr' folder within workspace
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
path: ${{ github.workspace }}/pr
- name: Render Helm chart for each scenario in the base branch
run: |
cd ${{ github.workspace }}/base/production/helm/loki
if [ ! -d "scenarios" ]; then
echo "Directory with the scenarios does not exist in base branch, skipping rendering them."
exit 0
fi
helm dependency build
for file in scenarios/*.yaml; do
echo "rendering scenario $(file)"
schenario_folder=${{ github.workspace }}/output/base/$(basename $file .yaml)
mkdir $schenario_folder
helm template loki-test-chart-name . -f $file --output-dir $schenario_folder
done
- name: Render Helm chart for each scenario in the PR branch
run: |
cd ${{ github.workspace }}/pr/production/helm/loki
# Check if the scenarios folder exists
if [ ! -d "scenarios" ]; then
echo "PR looks outdated because PRs branch does not have the scenarios, copying them from the base branch."
cp -r ${{ github.workspace }}/base/production/helm/loki/scenarios ./scenarios
fi
helm dependency build
for file in scenarios/*.yaml; do
echo "rendering scenario $(file)"
schenario_folder=${{ github.workspace }}/output/pr/$(basename $file .yaml)
mkdir $schenario_folder
helm template loki-test-chart-name . -f $file --output-dir $schenario_folder
done
- name: Calculate the diff between base and PR rendered manifests for each scenario
run: |
cd ${{ github.workspace }}/pr/production/helm/loki
for scenario_file in scenarios/*.yaml; do
added_files='[]'
modified_files='[]'
removed_files='[]'
scenario_name=$(basename $scenario_file .yaml)
base_branch_dir=${{ github.workspace }}/output/base/$scenario_name
pr_branch_dir=${{ github.workspace }}/output/pr/$scenario_name
echo "Comparing directories: $base_branch_dir and $pr_branch_dir"
# Find all files in the left and right directories
base_branch_files=$(if [[ -d "$base_branch_dir" ]]; then find "$base_branch_dir" -type f | sed "s|$base_branch_dir/||"; else echo ""; fi)
pr_branch_files=$(find "$pr_branch_dir" -type f | sed "s|$pr_branch_dir/||")
# Check for modified and removed files
for file in $base_branch_files; do
echo "check if file exists: $file"
if [[ -f "$pr_branch_dir/$file" ]]; then
echo "File exists in both directories, check if it is modified"
if ! diff -q "$base_branch_dir/$file" "$pr_branch_dir/$file" >/dev/null; then
echo "file is modified $file"
file_diff=$(diff -c "$base_branch_dir/$file" "$pr_branch_dir/$file" || true)
diff_obj=$(jq -n --arg file "$file" --arg diff "$file_diff" '{"filename": $file, "diff": $diff}')
# Append the new object to the JSON array using jq
modified_files=$(echo "$modified_files" | jq --argjson diff_obj "$diff_obj" '. += [$diff_obj]')
else
echo "file is not modified"
fi
else
echo "file is removed $file"
# File is missing in the PR directory
file_content=$(cat "$base_branch_dir/$file")
removed_obj=$(jq -n --arg filename "$file" --arg content "$file_content" '{"filename": $filename, "content": $content}')
# Append the new object to the JSON array using jq
removed_files=$(echo "$removed_files" | jq --argjson removed_obj "$removed_obj" '. += [$removed_obj]')
fi
done
# Check for added files in the right directory
for file in $pr_branch_files; do
if [[ ! -f "$base_branch_dir/$file" ]]; then
echo "added file detected"
# File is missing in the PR directory
file_content=$(cat "$pr_branch_dir/$file")
added_obj=$(jq -n --arg file "$file" --arg content "$file_content" '{"filename": $file, "content": $content}')
# Append the new object to the JSON array using jq
added_files=$(echo "$added_files" | jq --argjson added_obj "$added_obj" '. += [$added_obj]')
fi
done
scenario_output_dir="${{ github.workspace }}/output/$scenario_name"
mkdir $scenario_output_dir
echo $added_files > $scenario_output_dir/added_files.json
echo $modified_files > $scenario_output_dir/modified_files.json
echo $removed_files > $scenario_output_dir/removed_files.json
echo $removed_files
done
- name: Generate Markdown Summary
run: |
# Initialize the Markdown output file
output_file="${{ github.workspace }}/output/diff_summary.md"
echo "# Kubernetes Manifest Diff Summary" > $output_file
# Iterate over each scenario file
for file in ${{ github.workspace }}/pr/production/helm/loki/scenarios/*.yaml; do
scenario=$(basename "$file" .yaml)
echo "Processing scenario: $scenario"
# Read JSON data for added, modified, and removed files
added_files=$(cat ${{ github.workspace }}/output/$scenario/added_files.json)
modified_files=$(cat ${{ github.workspace }}/output/$scenario/modified_files.json)
removed_files=$(cat ${{ github.workspace }}/output/$scenario/removed_files.json)
# Count the number of added, modified, and removed files
num_added=$(echo "$added_files" | jq length)
num_modified=$(echo "$modified_files" | jq length)
num_removed=$(echo "$removed_files" | jq length)
# Create a header for the scenario
echo -e "\n<details><summary>Scenario: $scenario (Added: $num_added, Modified: $num_modified, Removed: $num_removed) </summary>\n" >> $output_file
echo -e "<p>\n\n" >> $output_file
# Add summary counts
echo -e "\n**Summary:**" >> $output_file
echo -e "\n- **Added:** $num_added" >> $output_file
echo -e "\n- **Modified:** $num_modified" >> $output_file
echo -e "\n- **Removed:** $num_removed" >> $output_file
# Add details for added files
echo -e "\n### Added Files" >> $output_file
if [[ "$num_added" -gt 0 ]]; then
echo "$added_files" | jq -c '.[]' | while read -r obj; do
filename=$(echo "$obj" | jq -r '.filename')
content=$(echo "$obj" | jq -r '.content')
echo -e "\n<details><summary>$filename</summary>" >> $output_file
echo -e "\n\`\`\`yaml\n$content\n\`\`\`\n</details>" >> $output_file
done
else
echo -e "\n_No added files_\n" >> $output_file
fi
# Add details for modified files
echo -e "\n### Modified Files" >> $output_file
if [[ "$num_modified" -gt 0 ]]; then
echo "$modified_files" | jq -c '.[]' | while read -r obj; do
filename=$(echo "$obj" | jq -r '.filename')
diff=$(echo "$obj" | jq -r '.diff')
echo -e "\n<details><summary>$filename</summary>" >> $output_file
echo -e "\n\`\`\`diff\n$diff\n\`\`\`\n</details>" >> $output_file
done
else
echo -e "\n_No modified files_\n" >> $output_file
fi
# Add details for removed files
echo -e "\n### Removed Files" >> $output_file
if [[ "$num_removed" -gt 0 ]]; then
echo "$removed_files" | jq -c '.[]' | while read -r obj; do
filename=$(echo "$obj" | jq -r '.filename')
content=$(echo "$obj" | jq -r '.content')
echo -e "\n<details><summary>$filename</summary>" >> $output_file
echo -e "\n\`\`\`yaml\n$content\n\`\`\`\n</details>" >> $output_file
done
else
echo -e "\n_No removed files_\n" >> $output_file
fi
# close <p> and <details>
echo -e "\n\n</p>\n</details>" >> $output_file
done
- name: Post diff as PR comment
uses: marocchino/sticky-pull-request-comment@v2
with:
hide_and_recreate: true
hide_classify: "OUTDATED"
path: ${{ github.workspace }}/output/diff_summary.md
production/helm/loki/scenarios/README.md
+54 -10
View File
@@ -1,19 +1,63 @@
These scenarios are used by Github Workflow: [Publish Rendered Helm Chart Diff](../../../../.github/workflows/helm-loki-ci.yml).
# Loki Helm Scenarios
Each scenario is used as the values file for the Loki Helm chart to render Kubernetes manifests in `base` and `PR's` branch to compare the content and report the diff on Pull Request as a comment([example](https://github.com/grafana/loki/pull/14127#issuecomment-2348360828)). It gives the ability to the reviewer to understand how the changes in the chart modify resulting manifests.
These scenarios are used by Github Workflow: [Publish Rendered Helm Chart Diff](../../../../.github/workflows/helm-diff-ci.yml).
![img.png](images/img.png)
Each scenario is used in a different job execution that will be used to deploy loki inside a K3D cluster in our github action workflow.
The workflow reports three types of changes for each scenario:
We deploy the scenario with the latest release and then we execute a helm diff with the current version in the workspace, the diff between the release deployed will be posted in the pull request inside a comment like [this](https://github.com/grafana/loki/pull/15734#issuecomment-2592439539) making clear to review.
1. Added files - the manifests that are added in the current PR and that did not exist in `base` branch.
>*NOTE*: the helm diff output file will be available for each scenario inside github action to download for 2 days, after this you may need to re-run the job if you would like to download the output files.
![added.png](images/added.png)
## Run scenarios locally
All this process that we run in the CI can be done locally, the following steps would explain how.
2. Modified files - the manifests that exist in both branches but the changes in PRs branch modify them.
![modified.png](images/modified.png)
## Requirements
3. Removed files - the manifests that exist in `base` branch but do not exist in PRs branch.
To run locally you will need the following tools in your local environment
![removed.png](images/removed.png)
* k3d or any kubernetes cluster
* helm
* [helm-diff plugin](https://github.com/databus23/helm-diff)
## Run
Make sure that you are pointing to the kubernetes cluster that you want to apply the chart and validate.
Create a `${scenario}-values.yaml` file with the configuration that you would like to validate.
Deploy in your kubernetes cluster the latest released version of the helm chart with your config file:
```shell
helm install --create-namespace loki-release grafana/loki -f ${scenario}-values.yaml
```
Then run the helm diff plugin to compare the local helm chart to see the upgrade changes that will happen:
```shell
HELM_DIFF_USE_UPGRADE_DRY_RUN: true helm diff upgrade loki-release -f ${scenario}-values.yaml production/helm/loki
```
The helm diff plugin will compare all manifests that are created in your local development kubernetes cluster and the generated by helm upgrade operation, the output will be printed in your terminal.
### Configs for CSP specific
To compare the changes specifically for a cloud provider the process is similar, the main difference that you will need to have access to the kubernetes cluster with the right permissions inside the cloud provider.
In case that is not possible, the quick validation can be done in a different way, instead of deploy to the kubernetes cluster, we generate the manifests with helm like this:
```shell
helm template loki-release grafana/loki -f ${scenario}-values.yaml > release-manifest.yaml
```
Then we make the same process with local chart version
```shell
helm template loki-release production/helm/loki -f ${scenario}-values.yaml > current-manifest.yaml
```
As the last step you need to run a diff between both files:
```shell
diff current-manifest.yaml release-manifest.yaml
```