diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 0000000..46678ca --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,126 @@ +name: Bug +description: Report a runtime failure (forge/cast), unexpected behavior, a Casting config issue (merge, defaults, schema), or a Molding config issue (defaults, template rendering). +title: "[bug] " +labels: ["bug"] +body: + - type: dropdown + id: category + attributes: + label: Category + description: What kind of issue is this? + options: + - Runtime (forge/cast failure, unexpected behavior) + - Casting config (merge behavior, missing defaults, schema) + - Molding config (defaults, template rendering, molding-specific) + validations: + required: true + + - type: textarea + id: description + attributes: + label: Describe the defect + description: | + Runtime: what went wrong, what command failed. + Casting config: describe the merge/forge behavior, missing defaults, or schema confusion. + Molding config: describe defaults, template rendering, or molding-specific issues. + validations: + required: true + + - type: textarea + id: casting_config + attributes: + label: Casting configuration + description: Paste your `casting.yaml` here (remove any sensitive values). Required for Casting/Molding config issues. + render: yaml + validations: + required: false + + - type: textarea + id: steps + attributes: + label: Steps to reproduce + description: For Casting/Molding config issues, describe how to trigger the problem (e.g., run forge with this config). + placeholder: | + 1. + 2. + 3. + validations: + required: false + + - type: textarea + id: expected + attributes: + label: Expected behavior + description: What did you expect Foundry to do? For Casting/Molding config, describe the expected merge/forge result. + validations: + required: true + + - type: textarea + id: actual + attributes: + label: Actual behavior + description: What actually happened? Include error output or actual `pours/` output. + render: shell + validations: + required: true + + - type: textarea + id: debug_output + attributes: + label: Command output + description: Run with `--debug` and paste the full output below (helpful for runtime issues). + placeholder: "foundryctl --debug <command> -f casting.yaml" + render: shell + validations: + required: false + + - type: input + id: foundry_version + attributes: + label: Foundry version + placeholder: "e.g., v0.1.0" + validations: + required: true + + - type: input + id: os_arch + attributes: + label: OS / Architecture + placeholder: "e.g., macOS arm64, Ubuntu 22.04 amd64" + validations: + required: true + + - type: dropdown + id: deployment_mode + attributes: + label: Deployment mode + options: + - docker + - systemd + - render + - kubernetes + - railway + - render + validations: + required: false + + - type: dropdown + id: flavor + attributes: + label: Flavor + options: + - compose + - binary + - blueprint + - kustomize + - swarm + validations: + required: false + + - type: textarea + id: additional_context + attributes: + label: Additional context + description: Screenshots, links, related docs, or workarounds tried. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/casting.yml b/.github/ISSUE_TEMPLATE/casting.yml new file mode 100644 index 0000000..09175ba --- /dev/null +++ b/.github/ISSUE_TEMPLATE/casting.yml @@ -0,0 +1,92 @@ +name: Casting +description: Request support for a new casting (e.g., Kubernetes, Nomad, a new cloud provider). +title: "[casting] <title>" +labels: ["casting", "enhancement"] +body: + - type: input + id: platform + attributes: + label: Deployment platform + description: What new deployment platform should Foundry support? + placeholder: "e.g., AWS, GCP, Render, Railway etc." + validations: + required: false + + - type: input + id: mode + attributes: + label: Deployment mode + description: What new deployment mode should Foundry support + placeholder: "e.g., Kubernetes, Docker, Systemd" + validations: + required: true + + - type: input + id: flavour + attributes: + label: Deployment flavor + description: What new deployment mode should Foundry support + placeholder: "e.g., Kustomize, Compose, Binary" + validations: + required: true + + - type: textarea + id: why + attributes: + label: Why this Casting? + description: Describe the use case. Who benefits and what environment does it target? + validations: + required: true + + - type: textarea + id: casting_config + attributes: + label: Proposed Casting configuration + description: Show how a user would configure this deployment mode in their `casting.yaml`. + render: yaml + placeholder: | + apiVersion: v1alpha1 + metadata: + name: signoz + spec: + deployment: + mode: <new-mode> + flavor: <new-flavor> + validations: + required: false + + - type: textarea + id: pours_structure + attributes: + label: Expected Pours structure + description: What files would be generated under `pours/` for this deployment mode? + placeholder: | + pours/ + └── deployment/ + └── ... + validations: + required: false + + - type: textarea + id: tooling + attributes: + label: Tooling requirements + description: What tools or CLIs need to be present (checked by `foundryctl gauge`)? + validations: + required: false + + - type: textarea + id: references + attributes: + label: References + description: Links to platform docs, examples, or prior art. + validations: + required: false + + - type: textarea + id: additional_context + attributes: + label: Additional context + description: Any constraints, limitations, or relevant discussion. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..6f4da74 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: SigNoz Slack Community + url: https://signoz.io/slack + about: Ask questions and get help from the SigNoz community. + - name: SigNoz Documentation + url: https://signoz.io/docs/ + about: Read the SigNoz docs for guides and reference material. diff --git a/.github/ISSUE_TEMPLATE/distribution.yml b/.github/ISSUE_TEMPLATE/distribution.yml new file mode 100644 index 0000000..8c88251 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/distribution.yml @@ -0,0 +1,48 @@ +name: Distribution +description: Request a new way to install or distribute foundryctl. +title: "[distribution] <title>" +labels: ["distribution", "enhancement"] +body: + - type: input + id: platform + attributes: + label: Target platforms + description: What OS and architectures should this support? + placeholder: "e.g., Linux (amd64, arm64), macOS (amd64, arm64), Windows" + validations: + required: false + + - type: textarea + id: install_command + attributes: + label: Proposed install command + description: Show how a user would install foundryctl with this method. + render: bash + placeholder: | + curl -LsSf https://signoz.io/foundry/install.sh | sh + validations: + required: false + + - type: textarea + id: motivation + attributes: + label: Motivation + description: Who benefits and what problem does it solve over existing methods? + validations: + required: true + + - type: textarea + id: additional_context + attributes: + label: Additional Context + description: Implementation notes, nice-to-haves, or any other relevant context. + validations: + required: false + + - type: textarea + id: references + attributes: + label: References + description: Links to similar installers, package registry docs, or prior art. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml new file mode 100644 index 0000000..cd29a82 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -0,0 +1,44 @@ +name: Feature Request +description: Suggest a new capability, improvement, or CLI enhancement for Foundry. +title: "[enhancement] <title>" +labels: ["enhancement"] +body: + - type: textarea + id: summary + attributes: + label: Summary + description: A short, one-line description of what you'd like Foundry to do. + validations: + required: true + + - type: textarea + id: motivation + attributes: + label: Motivation + description: Why is this useful? What problem does it solve or workflow does it improve? + validations: + required: true + + - type: textarea + id: proposed_solution + attributes: + label: Proposed solution + description: Describe the behavior you'd like. Include example commands, flags, or config snippets if relevant. + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: Have you tried working around this? Are there other approaches you considered? + validations: + required: false + + - type: textarea + id: additional_context + attributes: + label: Additional context + description: Any related issues, links, or screenshots. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/general.yml b/.github/ISSUE_TEMPLATE/general.yml new file mode 100644 index 0000000..a46f4c3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/general.yml @@ -0,0 +1,19 @@ +name: General +description: For issues that don't fit the Bug, Feature Request, or Casting templates. +labels: ["uncategorised"] +body: + - type: textarea + id: description + attributes: + label: Description + description: Describe your issue, question, or suggestion in detail. + validations: + required: true + + - type: textarea + id: additional_context + attributes: + label: Additional context + description: Any relevant links, screenshots, logs, or references. + validations: + required: false diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ee4cdab..4516815 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -8,7 +8,37 @@ permissions: contents: write id-token: write +env: + MAKE: make --no-print-directory --makefile=.primus/src/make/main.mk + jobs: + prepare: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.build-info.outputs.version }} + hash: ${{ steps.build-info.outputs.hash }} + steps: + - name: self-checkout + uses: actions/checkout@v4 + - id: token + name: github-token-gen + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.PRIMUS_APP_ID }} + private-key: ${{ secrets.PRIMUS_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + - name: primus-checkout + uses: actions/checkout@v4 + with: + repository: signoz/primus + ref: main + path: .primus + token: ${{ steps.token.outputs.token }} + - name: build-info + id: build-info + run: | + echo "version=$($MAKE info-version)" >> $GITHUB_OUTPUT + echo "hash=$($MAKE info-commit-short)" >> $GITHUB_OUTPUT docker: uses: o11y/primus.workflows/.github/workflows/go-build.yaml@main secrets: inherit @@ -16,7 +46,11 @@ jobs: PRIMUS_REF: main GO_BUILD_CONTEXT: ./cmd/foundryctl GO_VERSION: 1.25 - GO_BUILD_FLAGS: "" + GO_BUILD_FLAGS: >- + -ldflags='-s -w + -X github.com/signoz/foundry/internal/version.version=${{ needs.prepare.outputs.version }} + -X github.com/signoz/foundry/internal/version.hash=${{ needs.prepare.outputs.hash }} + -X github.com/signoz/foundry/internal/ledger.key=YPscRxkyHv2Fn0aOucmLrMYBBVAC1AFu' GO_NAME: foundryctl DOCKER_BASE_IMAGES: '{"distroless":"gcr.io/distroless/base:latest"}' DOCKER_DOCKERFILE_PATH: Dockerfile.multi-arch diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3390c36..10fe74a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,6 +18,14 @@ jobs: with: PRIMUS_REF: main GO_VERSION: 1.25 + sh-fmt: + if: | + (github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork && github.event.pull_request.user.login != 'dependabot[bot]' && ! contains(github.event.pull_request.labels.*.name, 'safe-to-test')) || + (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe-to-test')) + uses: signoz/primus.workflows/.github/workflows/sh-fmt.yaml@main + secrets: inherit + with: + PRIMUS_REF: main lint: if: | (github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork && github.event.pull_request.user.login != 'dependabot[bot]' && ! contains(github.event.pull_request.labels.*.name, 'safe-to-test')) || @@ -27,6 +35,14 @@ jobs: with: PRIMUS_REF: main GO_VERSION: 1.25 + sh-lint: + if: | + (github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork && github.event.pull_request.user.login != 'dependabot[bot]' && ! contains(github.event.pull_request.labels.*.name, 'safe-to-test')) || + (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe-to-test')) + uses: signoz/primus.workflows/.github/workflows/sh-lint.yaml@main + secrets: inherit + with: + PRIMUS_REF: main test: if: | (github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork && github.event.pull_request.user.login != 'dependabot[bot]' && ! contains(github.event.pull_request.labels.*.name, 'safe-to-test')) || @@ -60,8 +76,8 @@ jobs: go-version: "1.25" - name: gen-examples run: | - go run cmd/foundryctl/*.go gen examples - git diff --compact-summary --exit-code || (echo; echo "Unexpected difference in docs/examples/ directory. Run go run cmd/foundryctl/*.go gen locally and commit."; exit 1) + make gen-examples + git diff --compact-summary --exit-code || (echo; echo "Unexpected difference in docs/examples/. Run 'make gen-examples' locally and commit."; exit 1) schemas: if: | (github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork && github.event.pull_request.user.login != 'dependabot[bot]' && ! contains(github.event.pull_request.labels.*.name, 'safe-to-test')) || @@ -76,5 +92,21 @@ jobs: go-version: "1.25" - name: gen-schemas run: | - go run cmd/foundryctl/*.go gen schemas - git diff --compact-summary --exit-code || (echo; echo "Unexpected difference in docs/schemas/ directory. Run go run cmd/foundryctl/*.go gen locally and commit."; exit 1) \ No newline at end of file + make gen-schemas + git diff --compact-summary --exit-code || (echo; echo "Unexpected difference in docs/schemas/ directory. Run 'make gen-schemas' locally and commit."; exit 1) + docs: + if: | + (github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork && github.event.pull_request.user.login != 'dependabot[bot]' && ! contains(github.event.pull_request.labels.*.name, 'safe-to-test')) || + (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe-to-test')) + runs-on: ubuntu-latest + steps: + - name: self-checkout + uses: actions/checkout@v4 + - name: go-install + uses: actions/setup-go@v5 + with: + go-version: "1.25" + - name: gen-docs + run: | + make gen-docs + git diff --compact-summary --exit-code || (echo; echo "Unexpected difference in README.md. Run 'make gen-docs' locally and commit."; exit 1) diff --git a/.gitignore b/.gitignore index 0012152..a975184 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,6 @@ pours/ ### VSCode ### !.vscode/settings.json + +### Claude ### +CLAUDE.local.md diff --git a/.golangci.yml b/.golangci.yml index 2d80f03..3fe9055 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -24,3 +24,14 @@ linters: - identical sloglint: attr-only: true + forbidigo: + forbid: + - pattern: fmt.Errorf + - pattern: ^(fmt\.Print.*|print|println)$ + exclusions: + rules: + # ExitCode tests cover the wrap-chain path that fmt.Errorf("%w", ...) + # exercises in real callers; using fmt.Errorf here keeps the test honest. + - path: internal/errors/error_test\.go$ + linters: + - forbidigo diff --git a/.goreleaser.yaml b/.goreleaser.yaml index b259452..1a2221c 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -24,6 +24,11 @@ builds: goarm64: - v8.0 mod_timestamp: "{{ .CommitTimestamp }}" + ldflags: + - -s -w + - -X github.com/signoz/foundry/internal/version.version=v{{ .Version }} + - -X github.com/signoz/foundry/internal/version.hash={{ .ShortCommit }} + - -X github.com/signoz/foundry/internal/ledger.key=YPscRxkyHv2Fn0aOucmLrMYBBVAC1AFu tags: - timetzdata diff --git a/.versions/debian b/.versions/debian new file mode 100644 index 0000000..950f88a --- /dev/null +++ b/.versions/debian @@ -0,0 +1,16 @@ +#### Auto generated by make docker-version-debian. DO NOT EDIT! #### +amd64=8af0e5095f9964007f5ebd11191dfe52dcb51bf3afa2c07f055fc5451b78ba0e +unknown=aa26de2aeba103c4f2232de5efe83d2b4522c26fec20c5848c9f16f2aa244b3f +arm=159390659db54d341957fe78ef829de76e5549d0f9fcbb777287bcfc6f6da2f8 +unknown=83be97aab0d6eb03517006eea2cd7ecece54c20d3acbfe312d0ff4505dd4c344 +arm=0a52c3389a9a13dbeb3f3183578c15dfd712e17dad4feacbe55d224d0f459afa +unknown=728313602cf4214e7e5fd2ab88fa68f2f9a8e11118ec24b04f9f05e93fd443f0 +arm64=ee5473f786ff8a4e03409277c276b459b29afa55a84268bef55342c4f705b7ad +unknown=35e1cb927bb44a86ca8799cfe3d482a9e54e31a3995c28887c28bed9c0338bdd +386=02274f94f52336abd6ab4a8471ea09966613910ebaeed622429dc7b4b780e804 +unknown=3cc496e7c7df2b166481306e36070238a7644af6b230646688ca3c6f675995a2 +mips64le=b5bfd550af6266602aaef6c249baebf2c9835a2d0984013a23471cd8954503ff +ppc64le=8e69db6097e3efc0901fc84af17ea906710ae90b5842d919723eda3518ea6811 +unknown=d1dc187105900becbeca525ab9ef08efa3c71cdb4b8c67a2151e3aadc1f57a48 +s390x=a54db69bf6707382949f777f5fb8d0651d34a1b8602111a41eee7533e4ce07d8 +unknown=57ae880cf756c23f9afd4ef46b418eaae9232ea48f7c994b5c81215b58f226a7 diff --git a/LICENSE b/LICENSE index abc9c74..d8ca4aa 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,659 @@ -MIT License + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 Copyright (c) 2026 Hanzo O11y Inc. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + Preamble -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +<https://www.gnu.org/licenses/>. diff --git a/Makefile b/Makefile index 6d3125f..d6828d8 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,37 @@ +.PHONY: clean gauge forge cast test gen-examples gen-schemas gen-docs docs + +FOUNDRYCTL := go run ./cmd/foundryctl +GOTMPL := go run go.opentelemetry.io/build-tools/gotmpl@latest +CASTINGS_JSON = $$(cat docs/examples/castings.json) +NO_LEDGER := "--no-ledger" + clean: - cd pours/deployment && docker compose -p dev down --remove-orphans --volumes + cd pours/deployment && docker compose down --remove-orphans --volumes cd ../.. rm -rf ./pours gauge: - go run ./cmd/foundryctl gauge --debug -f ./tmp/casting.yaml + $(FOUNDRYCTL) gauge --debug $(NO_LEDGER) -f ./tmp/casting.yaml forge: - go run ./cmd/foundryctl forge --debug -f ./tmp/casting.yaml + $(FOUNDRYCTL) forge --debug $(NO_LEDGER) -f ./tmp/casting.yaml cast: - go run ./cmd/foundryctl cast --debug -f ./tmp/casting.yaml + $(FOUNDRYCTL) cast --debug $(NO_LEDGER) -f ./tmp/casting.yaml test: make forge make docker -gen: - go run ./cmd/foundryctl/*.go gen +gen-examples: + $(FOUNDRYCTL) gen --debug $(NO_LEDGER) examples + +gen-schemas: + $(FOUNDRYCTL) gen --debug $(NO_LEDGER) schemas + +gen-docs: + $(FOUNDRYCTL) catalog --debug $(NO_LEDGER) --format json --output "docs/examples/castings.json" + $(GOTMPL) -b README.md.gotmpl -d "$(CASTINGS_JSON)" -o README.md + $(GOTMPL) -b docs/examples/README.md.gotmpl -d "$(CASTINGS_JSON)" -o docs/examples/README.md + +docs: gen-examples gen-schemas gen-docs diff --git a/README.md b/README.md index d6cc49e..5218e0f 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,9 @@ <a href="https://o11y.hanzo.ai" target="_blank"> <img alt="Foundry" src="https://github.com/user-attachments/assets/ef9a33f7-12d7-4c94-8908-0a02b22f0c18" width="100" height="100"> </a> - <br>Foundry -</h1> +</p> + +<h1 align="center" style="border-bottom: none">Foundry</h1> <p align="center"> <img alt="GitHub Release" src="https://img.shields.io/github/v/release/o11y/foundry?include_prereleases"> @@ -49,7 +50,7 @@ curl -L "https://github.com/Hanzo O11y/foundry/releases/latest/download/foundry_ tar -xzf foundry.tar.gz ``` -**macOS:** +See [Getting Started](docs/getting-started.md) for manual install options and PATH setup. ```bash curl -L "https://github.com/Hanzo O11y/foundry/releases/latest/download/foundry_darwin_$(uname -m | sed 's/x86_64/amd64/g' | sed 's/arm64/arm64/g').tar.gz" -o foundry.tar.gz @@ -89,7 +90,6 @@ spec: ```bash foundryctl cast -f casting.yaml ``` -## The Foundry Model Foundry uses a metalworking metaphor: you define a **Casting**, which contains **Moldings** (components), and Foundry **forges** them into **Pours** (generated files). @@ -140,49 +140,85 @@ A Casting is a complete Hanzo O11y deployment definition: one YAML file that Fou **Pours** are the generated deployment and configuration files. When you run `forge`, Foundry creates the `pours/` directory containing everything needed to run Hanzo O11y. ``` -pours/ -└── deployment/ - ├── compose.yaml - └── configs/ - ├── ingester/ - │ ├── ingester.yaml - │ └── opamp.yaml - ├── telemetrykeeper/ - │ └── keeper-0.yaml - └── telemetrystore/ - ├── config.yaml - └── functions.yaml + +-------------------------------------------------------------+ + | casting.yaml | + | your single deployment config | + +-----------------------------+-------------------------------+ + | + +---------------+---------------+ + | | | + v v v + +-----------+ +-----------+ +----------------+ + | gauge | | forge | | cast | + |-----------| |-----------| |----------------| + | validate | | generate | | gauge + forge | + | prereqs | | files | | + deploy | + +-----------+ +-----+-----+ +-------+--------+ + | | + v | + +----------------------------------+ | + | pours/ | | + |----------------------------------| | + | compose.yaml manifests/ | | + | values.yaml configs/ | | + | render.yaml *.tf.json | | + +-----------------+----------------+ | + | | + +----------+---------+ + v + +-------------------------------------------------------------+ + | SigNoz Running | + |-------------------------------------------------------------| + | Docker Compose - Swarm - Systemd - Kubernetes - ECS | + | Render - Railway - Coolify | + +-------------------------------------------------------------+ ``` +`foundryctl cast` runs the full pipeline (gauge + forge + deploy) in one step. + +| Term | What it means | +| --- | --- | +| **Casting** | Your deployment config. One YAML file describing what you want. [Learn more](docs/concepts/casting.md) | +| **Moldings** | The SigNoz components (ClickHouse, PostgreSQL, OTel Collector, etc.) that Foundry configures for you. [Learn more](docs/concepts/moldings.md) | +| **Pours** | The generated output files in `pours/`. Structure varies by deployment mode. See [examples](docs/examples/) | + +## Examples + +| Platform | Mode | Flavor | Example | +| --- | --- | --- | --- | +| - | docker | compose | [docker/compose](docs/examples/docker/compose/) | +| - | docker | swarm | [docker/swarm](docs/examples/docker/swarm/) | +| - | kubernetes | helm | [kubernetes/helm](docs/examples/kubernetes/helm/) | +| - | kubernetes | kustomize | [kubernetes/kustomize](docs/examples/kubernetes/kustomize/) | +| - | systemd | binary | [systemd/binary](docs/examples/systemd/binary/) | +| ecs | ec2 | terraform | [ecs/ec2/terraform](docs/examples/ecs/ec2/terraform/) | +| coolify | - | stack | [coolify/stack](docs/examples/coolify/stack/) | +| railway | - | template | [railway/template](docs/examples/railway/template/) | +| render | - | blueprint | [render/blueprint](docs/examples/render/blueprint/) | + ## CLI reference ``` -Usage: - foundryctl [command] +foundryctl [command] -Available Commands: - gauge Gauge whether required tools are available - forge Forge configuration and deployment files - cast Cast to the target environment - gen Generate example files for all supported deployments - help Help about any command +Commands: + gauge Validate required tools for your deployment mode + forge Generate deployment and configuration files + cast Full pipeline: gauge + forge + deploy + gen Generate example casting files for all modes Flags: - -d, --debug Enable debug mode - -f, --file string Path to the Casting configuration file (default "casting.yaml") - -p, --pours string Directory for Pours (default "./pours") - -h, --help Help for foundryctl + -d, --debug Enable debug logging + -f, --file string Casting file path (default "casting.yaml") + -p, --pours string Output directory (default "./pours") ``` -### gauge - -Validates that all required tools are installed for your deployment mode: - ```bash +# Validate tools foundryctl gauge -f casting.yaml -``` -### forge +# Generate files only +foundryctl forge -f casting.yaml Generates deployment and configuration files based on your Casting: @@ -197,21 +233,12 @@ Deploys Hanzo O11y to your target environment. Runs `gauge` and `forge` automati ```bash foundryctl cast -f casting.yaml -# Skip gauge check -foundryctl cast --no-gauge - -# Skip forge (use existing Pours) -foundryctl cast --no-forge -``` - -### gen - -Generates example Casting configurations for all supported deployment modes: - -```bash +# Generate examples for all deployment modes foundryctl gen ``` +See [CLI Reference](docs/reference/cli.md) for the full command reference with all flags and examples. + ## What's next - [How to write a casting](docs/casting.md): step-by-step guide to casting files diff --git a/README.md.gotmpl b/README.md.gotmpl new file mode 100644 index 0000000..e6ef12e --- /dev/null +++ b/README.md.gotmpl @@ -0,0 +1,167 @@ +<p align="center"> + <a href="https://signoz.io" target="_blank"> + <img alt="Foundry" src="https://github.com/user-attachments/assets/ef9a33f7-12d7-4c94-8908-0a02b22f0c18" width="100" height="100"> + </a> +</p> + +<h1 align="center" style="border-bottom: none">Foundry</h1> + +<p align="center"> +<img alt="GitHub Release" src="https://img.shields.io/github/v/release/signoz/foundry?include_prereleases"> + <a href="https://golang.org"><img src="https://img.shields.io/badge/Go-1.25+-blue.svg" alt="Go Version"></a> + +<p align="center">Foundry is a centralized hub for <a href="https://signoz.io">SigNoz</a> installation configurations and deployments: <strong>integrations for install</strong>. Select yours, configure, and run SigNoz.</p> + +## Overview + +Just as a metalworking foundry turns raw materials into finished products, Foundry forges your deployment from a single configuration and casts SigNoz to fit your environment. + +Foundry abstracts away the complexities of the installation process so you can spend time *using* SigNoz rather than *installing* it. + +<p align="center"> + <img + src="docs/assets/cli.gif" + alt="Foundry CLI demo" + width="900" + loading="lazy" + style="border-radius: 8px; border: 1px solid #30363d;" + /> +</p> + +## Features + +- **Multi-platform support**: Deploy SigNoz using Docker Compose, Systemd (bare metal), or Render for flexible installation across environments. +- **Single configuration file**: Configure your entire SigNoz stack with one concise file. +- **Automatic dependency management**: Handles inter-service dependencies +- **Tool validation**: Verify prerequisites before deployment + +## Quick start + +**1. Install foundryctl** + +```bash +curl -fsSL https://signoz.io/foundry.sh | bash +``` + +See [Getting Started](docs/getting-started.md) for manual install options and PATH setup. + +**2. Create a casting** + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + mode: docker + flavor: compose +``` + +**3. Deploy** + +```bash +foundryctl cast -f casting.yaml +``` + +## How it works + +``` + +-------------------------------------------------------------+ + | casting.yaml | + | your single deployment config | + +-----------------------------+-------------------------------+ + | + +---------------+---------------+ + | | | + v v v + +-----------+ +-----------+ +----------------+ + | gauge | | forge | | cast | + |-----------| |-----------| |----------------| + | validate | | generate | | gauge + forge | + | prereqs | | files | | + deploy | + +-----------+ +-----+-----+ +-------+--------+ + | | + v | + +----------------------------------+ | + | pours/ | | + |----------------------------------| | + | compose.yaml manifests/ | | + | values.yaml configs/ | | + | render.yaml *.tf.json | | + +-----------------+----------------+ | + | | + +----------+---------+ + v + +-------------------------------------------------------------+ + | SigNoz Running | + |-------------------------------------------------------------| + | Docker Compose - Swarm - Systemd - Kubernetes - ECS | + | Render - Railway - Coolify | + +-------------------------------------------------------------+ +``` + +`foundryctl cast` runs the full pipeline (gauge + forge + deploy) in one step. + +| Term | What it means | +| --- | --- | +| **Casting** | Your deployment config. One YAML file describing what you want. [Learn more](docs/concepts/casting.md) | +| **Moldings** | The SigNoz components (ClickHouse, PostgreSQL, OTel Collector, etc.) that Foundry configures for you. [Learn more](docs/concepts/moldings.md) | +| **Pours** | The generated output files in `pours/`. Structure varies by deployment mode. See [examples](docs/examples/) | + +## Examples + +| Platform | Mode | Flavor | Example | +| --- | --- | --- | --- | +{{- range .Castings }} +| {{ or .platform "-" }} | {{ or .mode "-" }} | {{ or .flavor "-" }} | [{{ .example }}](docs/examples/{{ .example }}/) | +{{- end }} + +## CLI reference + +``` +foundryctl [command] + +Commands: + gauge Validate required tools for your deployment mode + forge Generate deployment and configuration files + cast Full pipeline: gauge + forge + deploy + gen Generate example casting files for all modes + +Flags: + -d, --debug Enable debug logging + -f, --file string Casting file path (default "casting.yaml") + -p, --pours string Output directory (default "./pours") +``` + +```bash +# Validate tools +foundryctl gauge -f casting.yaml + +# Generate files only +foundryctl forge -f casting.yaml + +# Full deploy +foundryctl cast -f casting.yaml + +# Generate examples for all deployment modes +foundryctl gen +``` + +See [CLI Reference](docs/reference/cli.md) for the full command reference with all flags and examples. + +## What's next + +- [Getting Started](docs/getting-started.md) - install and deploy your first SigNoz instance +- [Concepts](docs/concepts/) - understand castings, moldings, and patches +- [Examples](docs/examples/) - deployment configurations for all supported platforms +- [Reference](docs/reference/) - CLI commands and casting file spec +- [SigNoz documentation](https://signoz.io/docs/) - learn more about SigNoz +- [SigNoz Slack](https://signoz.io/slack) - community and support + +## How can I get help? + +- **Issues**: [GitHub Issues](https://github.com/signoz/foundry/issues) +- **Documentation**: [SigNoz Docs](https://signoz.io/docs/) +- **Community**: [SigNoz Slack](https://signoz.io/slack) + +**Made with ❤️ for the SigNoz community** diff --git a/api/v1alpha1/casting.go b/api/v1alpha1/casting.go deleted file mode 100644 index 70e2c22..0000000 --- a/api/v1alpha1/casting.go +++ /dev/null @@ -1,92 +0,0 @@ -package v1alpha1 - -type Casting struct { - TypeVersion `json:",inline" yaml:",inline"` - - // Metadata of the casting configuration. - Metadata TypeMetadata `json:"metadata" yaml:"metadata" description:"Metadata of the casting configuration"` - - // Specification for the casting. - Spec CastingSpec `json:"spec" yaml:"spec" description:"Specification for the casting"` - - // Status of the casting. - Status CastingStatus `json:"status,omitzero" yaml:"status,omitempty" description:"Status of the casting"` -} - -type CastingSpec struct { - // Mode platform in which the platform will run. - Deployment TypeDeployment `json:"deployment" yaml:"deployment" description:"Deployment configuration for the platform"` - - // The configuration for the o11y molding. - O11y HanzoO11y `json:"o11y,omitzero" yaml:"o11y,omitempty" description:"The configuration for the HanzoO11y molding"` - - // The configuration for the telemetry store molding. - TelemetryStore TelemetryStore `json:"telemetrystore,omitzero" yaml:"telemetrystore,omitempty" description:"The configuration for the telemetry store molding"` - - // The configuration for the telemetry keeper molding. - TelemetryKeeper TelemetryKeeper `json:"telemetrykeeper,omitzero" yaml:"telemetrykeeper,omitempty" description:"The configuration for the telemetry keeper molding"` - - // The configuration for the meta store molding. - MetaStore MetaStore `json:"metastore,omitzero" yaml:"metastore,omitempty" description:"The configuration for the meta store molding"` - - // The configuration for the ingester molding. - Ingester Ingester `json:"ingester,omitzero" yaml:"ingester,omitempty" description:"The configuration for the ingester molding"` -} - -type CastingStatus struct { - // Checksum of the casting file. - Checksum string `json:"checksum" yaml:"checksum" description:"Checksum of the casting file"` -} - -func MergeCastingSpecAndStatus(base *Casting) error { - if err := base.Spec.O11y.Spec.MergeStatus(base.Spec.O11y.Status.MoldingStatus); err != nil { - return err - } - - if err := base.Spec.TelemetryStore.Spec.MergeStatus(base.Spec.TelemetryStore.Status.MoldingStatus); err != nil { - return err - } - - if err := base.Spec.TelemetryKeeper.Spec.MergeStatus(base.Spec.TelemetryKeeper.Status.MoldingStatus); err != nil { - return err - } - - if err := base.Spec.MetaStore.Spec.MergeStatus(base.Spec.MetaStore.Status.MoldingStatus); err != nil { - return err - } - - if err := base.Spec.Ingester.Spec.MergeStatus(base.Spec.Ingester.Status.MoldingStatus); err != nil { - return err - } - - return nil -} - -func DefaultCasting() Casting { - return Casting{ - TypeVersion: TypeVersion{ - APIVersion: "v1alpha1", - }, - Metadata: TypeMetadata{ - Name: "o11y", - }, - Spec: CastingSpec{ - O11y: DefaultHanzoO11y(), - TelemetryStore: DefaultTelemetryStore(), - TelemetryKeeper: DefaultTelemetryKeeper(), - MetaStore: DefaultMetaStore(), - Ingester: DefaultIngester(), - }, - } -} - -func ExampleCasting() Casting { - return Casting{ - TypeVersion: TypeVersion{ - APIVersion: "v1alpha1", - }, - Metadata: TypeMetadata{ - Name: "o11y", - }, - } -} diff --git a/api/v1alpha1/casting.schema.json b/api/v1alpha1/casting.schema.json new file mode 100644 index 0000000..5607888 --- /dev/null +++ b/api/v1alpha1/casting.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Foundry Casting", + "oneOf": [ + { + "$ref": "installation/casting.schema.json" + }, + { + "$ref": "collectionagent/casting.schema.json" + } + ] +} \ No newline at end of file diff --git a/api/v1alpha1/casting_kind.go b/api/v1alpha1/casting_kind.go new file mode 100644 index 0000000..3d71090 --- /dev/null +++ b/api/v1alpha1/casting_kind.go @@ -0,0 +1,87 @@ +package v1alpha1 + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/swaggest/jsonschema-go" + "go.yaml.in/yaml/v3" +) + +var _ yaml.Marshaler = (*Kind)(nil) +var _ yaml.Unmarshaler = (*Kind)(nil) +var _ json.Marshaler = (*Kind)(nil) +var _ json.Unmarshaler = (*Kind)(nil) +var _ fmt.Stringer = (*Kind)(nil) +var _ jsonschema.Enum = (*Kind)(nil) + +var ( + KindInstallation Kind = Kind{s: "Installation"} + KindCollectionAgent Kind = Kind{s: "CollectionAgent"} +) + +// Kind discriminates between top-level casting resource types. +// An empty/missing kind unmarshals to KindInstallation for backwards compatibility +// with casting files written before kind was introduced. +type Kind struct { + s string +} + +func (kind Kind) String() string { + return kind.s +} + +func Kinds() []Kind { + return []Kind{KindInstallation, KindCollectionAgent} +} + +func (kind Kind) MarshalJSON() ([]byte, error) { + return json.Marshal(kind.String()) +} + +func (kind *Kind) UnmarshalJSON(text []byte) error { + var str string + if err := json.Unmarshal(text, &str); err != nil { + return err + } + + return kind.UnmarshalText([]byte(str)) +} + +func (kind *Kind) UnmarshalText(text []byte) error { + if len(text) == 0 { + *kind = KindInstallation + return nil + } + + for _, available := range Kinds() { + if available.String() == string(text) { + *kind = available + return nil + } + } + + return errors.New("invalid kind: " + string(text)) +} + +func (kind Kind) MarshalText() ([]byte, error) { + return []byte(kind.String()), nil +} + +func (kind *Kind) UnmarshalYAML(node *yaml.Node) error { + return kind.UnmarshalText([]byte(node.Value)) +} + +func (kind Kind) MarshalYAML() (any, error) { + return kind.String(), nil +} + +func (kind Kind) Enum() []any { + kinds := []any{} + for _, kind := range Kinds() { + kinds = append(kinds, kind.String()) + } + + return kinds +} diff --git a/api/v1alpha1/casting_meta.go b/api/v1alpha1/casting_meta.go new file mode 100644 index 0000000..483d2d7 --- /dev/null +++ b/api/v1alpha1/casting_meta.go @@ -0,0 +1,17 @@ +package v1alpha1 + +// CastingMeta carries the apiVersion, kind, metadata, and status fields +// shared by every casting kind. Per-Kind Casting structs embed it inline. +type CastingMeta struct { + TypeVersion `json:",inline" yaml:",inline"` + Kind Kind `json:"kind" yaml:"kind" required:"true" description:"Kind of the casting resource."` + Metadata TypeMetadata `json:"metadata" yaml:"metadata" required:"true" description:"Metadata of the casting configuration"` + Status Status `json:"status,omitzero" yaml:"status,omitempty" description:"Status of the casting"` + _ struct{} `additionalProperties:"false"` +} + +// Status carries the casting file's checksum. +type Status struct { + Checksum string `json:"checksum" yaml:"checksum" description:"Checksum of the casting file"` + _ struct{} `additionalProperties:"false"` +} diff --git a/api/v1alpha1/collectionagent/casting.go b/api/v1alpha1/collectionagent/casting.go new file mode 100644 index 0000000..fbe567f --- /dev/null +++ b/api/v1alpha1/collectionagent/casting.go @@ -0,0 +1,75 @@ +package collectionagent + +import ( + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/internal/domain" +) + +// Casting is the CollectionAgent kind. +type Casting struct { + v1alpha1.CastingMeta `json:",inline" yaml:",inline"` + Spec Spec `json:"spec" yaml:"spec" required:"true" description:"CollectionAgent specification"` + _ struct{} `additionalProperties:"false"` +} + +// Spec is the CollectionAgent-specific configuration. +type Spec struct { + Deployment v1alpha1.TypeDeployment `json:"deployment" yaml:"deployment" required:"true" description:"Deployment configuration for the platform"` + Patches []v1alpha1.PatchEntry `json:"patches,omitempty" yaml:"patches,omitempty" description:"Patch operations to apply to generated materials"` + Collector Collector `json:"collector,omitzero" yaml:"collector,omitempty" description:"The configuration for the collector molding"` + _ struct{} `additionalProperties:"false"` +} + +var _ v1alpha1.Machinery = (*Casting)(nil) + +// Default returns a CollectionAgent with the collector molding initialised +// from its default. +func Default() *Casting { + return &Casting{ + CastingMeta: v1alpha1.CastingMeta{ + TypeVersion: v1alpha1.TypeVersion{APIVersion: "v1alpha1"}, + Kind: v1alpha1.KindCollectionAgent, + Metadata: v1alpha1.TypeMetadata{Name: "signoz"}, + }, + Spec: Spec{ + Collector: DefaultCollector(), + }, + } +} + +// Example returns a minimal CollectionAgent; the forge pipeline fills in +// defaults. +func Example() *Casting { + return &Casting{ + CastingMeta: v1alpha1.CastingMeta{ + TypeVersion: v1alpha1.TypeVersion{APIVersion: "v1alpha1"}, + Kind: v1alpha1.KindCollectionAgent, + Metadata: v1alpha1.TypeMetadata{Name: "signoz"}, + }, + } +} + +// Kind reports the casting kind. Shadows the embedded CastingMeta.Kind field; +// the field stays reachable as c.CastingMeta.Kind. +func (c *Casting) Kind() v1alpha1.Kind { + return v1alpha1.KindCollectionAgent +} + +// MergeStatusIntoSpec folds the collector molding's Status into its Spec. +func (c *Casting) MergeStatusIntoSpec() error { + if err := c.Spec.Collector.Spec.MergeStatus(c.Spec.Collector.Status.MoldingStatus); err != nil { + return err + } + return nil +} + +// TrackableProperties returns analytics tags for the casting. +func (c *Casting) TrackableProperties() domain.Properties { + return domain.NewProperties(). + Set("kind", v1alpha1.KindCollectionAgent.String()). + Set("platform", c.Spec.Deployment.Platform.String()). + Set("mode", c.Spec.Deployment.Mode.String()). + Set("flavor", c.Spec.Deployment.Flavor.String()). + Set("patches_count", len(c.Spec.Patches)). + Set("collector_kind", c.Spec.Collector.Kind.String()) +} diff --git a/api/v1alpha1/collectionagent/casting.schema.json b/api/v1alpha1/collectionagent/casting.schema.json new file mode 100644 index 0000000..8998b54 --- /dev/null +++ b/api/v1alpha1/collectionagent/casting.schema.json @@ -0,0 +1,384 @@ +{ + "required": [ + "apiVersion", + "kind", + "metadata", + "spec" + ], + "additionalProperties": false, + "definitions": { + "CollectionagentCollector": { + "additionalProperties": false, + "properties": { + "kind": { + "$ref": "#/definitions/CollectionagentCollectorKind", + "description": "Kind of the collector to use", + "examples": [ + "agent" + ] + }, + "spec": { + "$ref": "#/definitions/V1Alpha1MoldingSpec", + "description": "Specification for the collector" + }, + "status": { + "$ref": "#/definitions/CollectionagentCollectorStatus", + "description": "Status of the collector" + } + }, + "type": "object" + }, + "CollectionagentCollectorKind": { + "enum": [ + "agent" + ], + "type": "string" + }, + "CollectionagentCollectorStatus": { + "additionalProperties": false, + "properties": { + "config": { + "$ref": "#/definitions/V1Alpha1TypeConfig", + "description": "Configuration for the molding" + }, + "env": { + "description": "Environment variables for the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "extras": { + "description": "Extra information about the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + } + }, + "type": "object" + }, + "CollectionagentSpec": { + "required": [ + "deployment" + ], + "additionalProperties": false, + "properties": { + "collector": { + "$ref": "#/definitions/CollectionagentCollector", + "description": "The configuration for the collector molding" + }, + "deployment": { + "$ref": "#/definitions/V1Alpha1TypeDeployment", + "description": "Deployment configuration for the platform" + }, + "patches": { + "description": "Patch operations to apply to generated materials", + "items": { + "$ref": "#/definitions/V1Alpha1PatchEntry" + }, + "type": "array" + } + }, + "type": "object" + }, + "V1Alpha1Flavor": { + "enum": [ + "compose", + "swarm", + "binary", + "kustomize", + "helm", + "blueprint", + "stack", + "template", + "terraform" + ], + "type": "string" + }, + "V1Alpha1Kind": { + "enum": [ + "CollectionAgent" + ], + "type": "string" + }, + "V1Alpha1Mode": { + "enum": [ + "docker", + "systemd", + "kubernetes", + "ec2" + ], + "type": "string" + }, + "V1Alpha1MoldingSpec": { + "additionalProperties": false, + "properties": { + "cluster": { + "$ref": "#/definitions/V1Alpha1TypeCluster", + "description": "Cluster configuration for the molding" + }, + "config": { + "$ref": "#/definitions/V1Alpha1TypeConfig", + "description": "Configuration for the molding" + }, + "enabled": { + "description": "Whether the molding is enabled", + "default": true, + "type": [ + "null", + "boolean" + ] + }, + "env": { + "description": "Environment variables for the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "image": { + "description": "Container image of the molding", + "examples": [ + "signoz/signoz:latest" + ], + "pattern": "^[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*(:[a-zA-Z0-9._-]+)?(@sha256:[a-f0-9]{64})?$", + "type": "string" + }, + "version": { + "description": "The version of the molding to use", + "examples": [ + "latest" + ], + "type": "string" + } + }, + "type": "object" + }, + "V1Alpha1PatchEntry": { + "required": [ + "target", + "operations" + ], + "additionalProperties": false, + "properties": { + "operations": { + "description": "JSON Patch (RFC 6902) operations to apply. Used by the jsonpatch driver.", + "items": { + "$ref": "#/definitions/V1Alpha1PatchOperation" + }, + "minItems": 1, + "type": [ + "array", + "null" + ] + }, + "target": { + "description": "Target output file to patch", + "examples": [ + "compose.yaml", + "signoz/deployment.yaml", + "values.yaml", + "telemetrystore/telemtrystore-clickhouse-0-*.yaml" + ], + "minLength": 1, + "type": "string" + }, + "type": { + "description": "Patch driver type. Defaults to jsonpatch.", + "default": "jsonpatch", + "examples": [ + "jsonpatch" + ], + "enum": [ + "", + "jsonpatch" + ], + "type": "string" + } + }, + "type": "object" + }, + "V1Alpha1PatchOperation": { + "required": [ + "op", + "path" + ], + "additionalProperties": false, + "properties": { + "from": { + "description": "Source JSON Pointer for move and copy operations", + "examples": [ + "/services/clickhouse/old_field" + ], + "pattern": "^/", + "type": "string" + }, + "op": { + "description": "JSON Patch (RFC 6902) operation type", + "enum": [ + "add", + "remove", + "replace", + "move", + "copy", + "test" + ], + "type": "string" + }, + "path": { + "description": "JSON Pointer (RFC 6901) to the target location", + "examples": [ + "/services/clickhouse/mem_limit" + ], + "pattern": "^/", + "type": "string" + }, + "value": { + "description": "Value for add, replace, or test operations" + } + }, + "type": "object" + }, + "V1Alpha1Platform": { + "enum": [ + "render", + "coolify", + "railway", + "ecs", + "aws", + "gcp", + "azure" + ], + "type": "string" + }, + "V1Alpha1Status": { + "additionalProperties": false, + "properties": { + "checksum": { + "description": "Checksum of the casting file", + "type": "string" + } + }, + "type": "object" + }, + "V1Alpha1TypeCluster": { + "additionalProperties": false, + "properties": { + "replicas": { + "description": "Number of replicas for the molding.", + "examples": [ + 1 + ], + "minimum": 0, + "type": [ + "null", + "integer" + ] + }, + "shards": { + "description": "Number of shards for the molding", + "examples": [ + 1 + ], + "minimum": 1, + "type": [ + "null", + "integer" + ] + } + }, + "type": "object" + }, + "V1Alpha1TypeConfig": { + "additionalProperties": false, + "properties": { + "data": { + "description": "Configuration data as key-value pairs.", + "additionalProperties": { + "type": "string" + }, + "type": "object" + } + }, + "type": "object" + }, + "V1Alpha1TypeDeployment": { + "additionalProperties": false, + "properties": { + "flavor": { + "$ref": "#/definitions/V1Alpha1Flavor", + "description": "Flavor of mode for the deployment" + }, + "mode": { + "$ref": "#/definitions/V1Alpha1Mode", + "description": "Type of installation method" + }, + "platform": { + "$ref": "#/definitions/V1Alpha1Platform", + "description": "Provider where an installation runs on" + } + }, + "type": "object" + }, + "V1Alpha1TypeMetadata": { + "required": [ + "name" + ], + "additionalProperties": false, + "properties": { + "annotations": { + "description": "Annotations is an unstructured key-value map for arbitrary metadata. Can be used to specify deployment-specific settings.", + "additionalProperties": { + "type": "string" + }, + "type": [ + "object", + "null" + ] + }, + "name": { + "description": "The name of this installation. This name is used to identify the installation.", + "default": "signoz", + "examples": [ + "signoz" + ], + "maxLength": 63, + "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", + "type": "string" + } + }, + "type": "object" + } + }, + "properties": { + "apiVersion": { + "description": "API Version of the configuration schema.", + "default": "v1alpha1", + "examples": [ + "v1alpha1" + ], + "enum": [ + "v1alpha1" + ], + "type": "string" + }, + "kind": { + "$ref": "#/definitions/V1Alpha1Kind", + "description": "Kind of the casting resource." + }, + "metadata": { + "$ref": "#/definitions/V1Alpha1TypeMetadata", + "description": "Metadata of the casting configuration" + }, + "spec": { + "$ref": "#/definitions/CollectionagentSpec", + "description": "CollectionAgent specification" + }, + "status": { + "$ref": "#/definitions/V1Alpha1Status", + "description": "Status of the casting" + } + }, + "type": "object" +} \ No newline at end of file diff --git a/api/v1alpha1/collectionagent/collector.go b/api/v1alpha1/collectionagent/collector.go new file mode 100644 index 0000000..0a46351 --- /dev/null +++ b/api/v1alpha1/collectionagent/collector.go @@ -0,0 +1,40 @@ +package collectionagent + +import "github.com/signoz/foundry/api/v1alpha1" + +type Collector struct { + // Kind of the collector to use. + Kind CollectorKind `json:"kind,omitzero" yaml:"kind,omitempty" description:"Kind of the collector to use" examples:"[\"agent\"]"` + + // Specification for the collector. + Spec v1alpha1.MoldingSpec `json:"spec" yaml:"spec" description:"Specification for the collector"` + + // Status of the collector. + Status CollectorStatus `json:"status" yaml:"status,omitempty" description:"Status of the collector"` + + _ struct{} `additionalProperties:"false"` +} + +type CollectorStatus struct { + v1alpha1.MoldingStatus `json:",inline" yaml:",inline"` + + _ struct{} `additionalProperties:"false"` +} + +func DefaultCollector() Collector { + return Collector{ + Kind: CollectorKindAgent, + Spec: v1alpha1.MoldingSpec{ + Enabled: v1alpha1.BoolPtr(true), + Cluster: v1alpha1.TypeCluster{ + Replicas: v1alpha1.IntPtr(1), + }, + Version: "v0.139.0", + Image: "otel/opentelemetry-collector-contrib:v0.139.0", + Env: map[string]string{}, + Config: v1alpha1.TypeConfig{ + Data: map[string]string{}, + }, + }, + } +} diff --git a/api/v1alpha1/collectionagent/collector_kind.go b/api/v1alpha1/collectionagent/collector_kind.go new file mode 100644 index 0000000..5dc0778 --- /dev/null +++ b/api/v1alpha1/collectionagent/collector_kind.go @@ -0,0 +1,81 @@ +package collectionagent + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/swaggest/jsonschema-go" + "go.yaml.in/yaml/v3" +) + +var _ yaml.Marshaler = (*CollectorKind)(nil) +var _ yaml.Unmarshaler = (*CollectorKind)(nil) +var _ json.Marshaler = (*CollectorKind)(nil) +var _ json.Unmarshaler = (*CollectorKind)(nil) +var _ fmt.Stringer = (*CollectorKind)(nil) +var _ jsonschema.Enum = (*CollectorKind)(nil) + +var ( + CollectorKindAgent CollectorKind = CollectorKind{s: "agent"} +) + +type CollectorKind struct { + s string +} + +func (kind CollectorKind) String() string { + return kind.s +} + +func CollectorKinds() []CollectorKind { + return []CollectorKind{CollectorKindAgent} +} + +func (kind CollectorKind) MarshalJSON() ([]byte, error) { + return json.Marshal(kind.String()) +} + +func (kind *CollectorKind) UnmarshalJSON(text []byte) error { + var str string + if err := json.Unmarshal(text, &str); err != nil { + return err + } + + return kind.UnmarshalText([]byte(str)) +} + +func (kind *CollectorKind) UnmarshalText(text []byte) error { + for _, availableKind := range CollectorKinds() { + if availableKind.String() == string(text) { + *kind = availableKind + return nil + } + } + if text == nil { + *kind = CollectorKind{s: ""} + return nil + } + return errors.New("invalid collector kind: " + string(text)) +} + +func (kind CollectorKind) MarshalText() ([]byte, error) { + return []byte(kind.String()), nil +} + +func (kind *CollectorKind) UnmarshalYAML(node *yaml.Node) error { + return kind.UnmarshalText([]byte(node.Value)) +} + +func (kind CollectorKind) MarshalYAML() (any, error) { + return kind.String(), nil +} + +func (kind CollectorKind) Enum() []any { + kinds := []any{} + for _, kind := range CollectorKinds() { + kinds = append(kinds, kind.String()) + } + + return kinds +} diff --git a/api/v1alpha1/collectionagent/schema.go b/api/v1alpha1/collectionagent/schema.go new file mode 100644 index 0000000..d01ba5e --- /dev/null +++ b/api/v1alpha1/collectionagent/schema.go @@ -0,0 +1,18 @@ +package collectionagent + +import ( + _ "embed" + + "github.com/google/jsonschema-go/jsonschema" + "github.com/signoz/foundry/api/v1alpha1" +) + +//go:embed casting.schema.json +var schemaJSON []byte + +var schema = v1alpha1.MustResolveSchema(schemaJSON) + +// Schema returns the resolved JSON schema for a CollectionAgent casting. +func Schema() *jsonschema.Resolved { + return schema +} diff --git a/api/v1alpha1/collectionagent/schema_test.go b/api/v1alpha1/collectionagent/schema_test.go new file mode 100644 index 0000000..8405f5f --- /dev/null +++ b/api/v1alpha1/collectionagent/schema_test.go @@ -0,0 +1,26 @@ +package collectionagent + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSchema(t *testing.T) { + t.Parallel() + assert.NotNil(t, Schema()) +} + +func TestSchemaValidatesDefault(t *testing.T) { + t.Parallel() + + contents, err := json.Marshal(Default()) + require.NoError(t, err) + + var payload map[string]any + require.NoError(t, json.Unmarshal(contents, &payload)) + + assert.NoError(t, Schema().Validate(payload)) +} diff --git a/api/v1alpha1/deployment_flavor.go b/api/v1alpha1/deployment_flavor.go new file mode 100644 index 0000000..e09b174 --- /dev/null +++ b/api/v1alpha1/deployment_flavor.go @@ -0,0 +1,101 @@ +package v1alpha1 + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/swaggest/jsonschema-go" + "go.yaml.in/yaml/v3" +) + +var _ yaml.Marshaler = (*Flavor)(nil) +var _ yaml.Unmarshaler = (*Flavor)(nil) +var _ json.Marshaler = (*Flavor)(nil) +var _ json.Unmarshaler = (*Flavor)(nil) +var _ fmt.Stringer = (*Flavor)(nil) +var _ jsonschema.Enum = (*Flavor)(nil) + +var ( + FlavorCompose Flavor = Flavor{s: "compose"} + FlavorSwarm Flavor = Flavor{s: "swarm"} + FlavorBinary Flavor = Flavor{s: "binary"} + FlavorKustomize Flavor = Flavor{s: "kustomize"} + FlavorHelm Flavor = Flavor{s: "helm"} + FlavorBlueprint Flavor = Flavor{s: "blueprint"} + FlavorStack Flavor = Flavor{s: "stack"} + FlavorTemplate Flavor = Flavor{s: "template"} + FlavorTerraform Flavor = Flavor{s: "terraform"} +) + +type Flavor struct { + s string +} + +func (flavor Flavor) String() string { + return flavor.s +} + +func Flavors() []Flavor { + return []Flavor{ + FlavorCompose, + FlavorSwarm, + FlavorBinary, + FlavorKustomize, + FlavorHelm, + FlavorBlueprint, + FlavorStack, + FlavorTemplate, + FlavorTerraform, + } +} + +func (flavor Flavor) MarshalJSON() ([]byte, error) { + return json.Marshal(flavor.String()) +} + +func (flavor *Flavor) UnmarshalJSON(text []byte) error { + var str string + if err := json.Unmarshal(text, &str); err != nil { + return err + } + + return flavor.UnmarshalText([]byte(str)) +} + +func (flavor *Flavor) UnmarshalText(text []byte) error { + for _, available := range Flavors() { + if available.String() == string(text) { + *flavor = available + return nil + } + } + + if len(text) == 0 { + *flavor = Flavor{s: ""} + return nil + } + + return errors.New("invalid deployment flavor: " + string(text)) +} + +func (flavor Flavor) MarshalText() ([]byte, error) { + return []byte(flavor.String()), nil +} + +func (flavor *Flavor) UnmarshalYAML(node *yaml.Node) error { + return flavor.UnmarshalText([]byte(node.Value)) +} + +func (flavor Flavor) MarshalYAML() (any, error) { + return flavor.String(), nil +} + +func (flavor Flavor) Enum() []any { + flavors := []any{} + for _, flavor := range Flavors() { + flavors = append(flavors, flavor.String()) + } + + return flavors +} diff --git a/api/v1alpha1/deployment_mode.go b/api/v1alpha1/deployment_mode.go new file mode 100644 index 0000000..a787efc --- /dev/null +++ b/api/v1alpha1/deployment_mode.go @@ -0,0 +1,86 @@ +package v1alpha1 + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/swaggest/jsonschema-go" + "go.yaml.in/yaml/v3" +) + +var _ yaml.Marshaler = (*Mode)(nil) +var _ yaml.Unmarshaler = (*Mode)(nil) +var _ json.Marshaler = (*Mode)(nil) +var _ json.Unmarshaler = (*Mode)(nil) +var _ fmt.Stringer = (*Mode)(nil) +var _ jsonschema.Enum = (*Mode)(nil) + +var ( + ModeDocker Mode = Mode{s: "docker"} + ModeSystemd Mode = Mode{s: "systemd"} + ModeKubernetes Mode = Mode{s: "kubernetes"} + ModeEC2 Mode = Mode{s: "ec2"} +) + +type Mode struct { + s string +} + +func (mode Mode) String() string { + return mode.s +} + +func Modes() []Mode { + return []Mode{ModeDocker, ModeSystemd, ModeKubernetes, ModeEC2} +} + +func (mode Mode) MarshalJSON() ([]byte, error) { + return json.Marshal(mode.String()) +} + +func (mode *Mode) UnmarshalJSON(text []byte) error { + var str string + if err := json.Unmarshal(text, &str); err != nil { + return err + } + + return mode.UnmarshalText([]byte(str)) +} + +func (mode *Mode) UnmarshalText(text []byte) error { + for _, available := range Modes() { + if available.String() == string(text) { + *mode = available + return nil + } + } + + if len(text) == 0 { + *mode = Mode{s: ""} + return nil + } + + return errors.New("invalid deployment mode: " + string(text)) +} + +func (mode Mode) MarshalText() ([]byte, error) { + return []byte(mode.String()), nil +} + +func (mode *Mode) UnmarshalYAML(node *yaml.Node) error { + return mode.UnmarshalText([]byte(node.Value)) +} + +func (mode Mode) MarshalYAML() (any, error) { + return mode.String(), nil +} + +func (mode Mode) Enum() []any { + modes := []any{} + for _, mode := range Modes() { + modes = append(modes, mode.String()) + } + + return modes +} diff --git a/api/v1alpha1/deployment_platform.go b/api/v1alpha1/deployment_platform.go new file mode 100644 index 0000000..7660bf3 --- /dev/null +++ b/api/v1alpha1/deployment_platform.go @@ -0,0 +1,97 @@ +package v1alpha1 + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/swaggest/jsonschema-go" + "go.yaml.in/yaml/v3" +) + +var _ yaml.Marshaler = (*Platform)(nil) +var _ yaml.Unmarshaler = (*Platform)(nil) +var _ json.Marshaler = (*Platform)(nil) +var _ json.Unmarshaler = (*Platform)(nil) +var _ fmt.Stringer = (*Platform)(nil) +var _ jsonschema.Enum = (*Platform)(nil) + +var ( + PlatformRender Platform = Platform{s: "render"} + PlatformCoolify Platform = Platform{s: "coolify"} + PlatformRailway Platform = Platform{s: "railway"} + PlatformECS Platform = Platform{s: "ecs"} + PlatformAWS Platform = Platform{s: "aws"} + PlatformGCP Platform = Platform{s: "gcp"} + PlatformAzure Platform = Platform{s: "azure"} +) + +type Platform struct { + s string +} + +func (platform Platform) String() string { + return platform.s +} + +func Platforms() []Platform { + return []Platform{ + PlatformRender, + PlatformCoolify, + PlatformRailway, + PlatformECS, + PlatformAWS, + PlatformGCP, + PlatformAzure, + } +} + +func (platform Platform) MarshalJSON() ([]byte, error) { + return json.Marshal(platform.String()) +} + +func (platform *Platform) UnmarshalJSON(text []byte) error { + var str string + if err := json.Unmarshal(text, &str); err != nil { + return err + } + + return platform.UnmarshalText([]byte(str)) +} + +func (platform *Platform) UnmarshalText(text []byte) error { + for _, available := range Platforms() { + if available.String() == string(text) { + *platform = available + return nil + } + } + + if len(text) == 0 { + *platform = Platform{s: ""} + return nil + } + + return errors.New("invalid deployment platform: " + string(text)) +} + +func (platform Platform) MarshalText() ([]byte, error) { + return []byte(platform.String()), nil +} + +func (platform *Platform) UnmarshalYAML(node *yaml.Node) error { + return platform.UnmarshalText([]byte(node.Value)) +} + +func (platform Platform) MarshalYAML() (any, error) { + return platform.String(), nil +} + +func (platform Platform) Enum() []any { + platforms := []any{} + for _, platform := range Platforms() { + platforms = append(platforms, platform.String()) + } + + return platforms +} diff --git a/api/v1alpha1/installation/casting.go b/api/v1alpha1/installation/casting.go new file mode 100644 index 0000000..9794b5b --- /dev/null +++ b/api/v1alpha1/installation/casting.go @@ -0,0 +1,99 @@ +package installation + +import ( + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/internal/domain" +) + +// Casting is the Installation kind. +type Casting struct { + v1alpha1.CastingMeta `json:",inline" yaml:",inline"` + Spec Spec `json:"spec" yaml:"spec" required:"true" description:"Installation specification"` + _ struct{} `additionalProperties:"false"` +} + +// Spec is the Installation-specific configuration. +type Spec struct { + Deployment v1alpha1.TypeDeployment `json:"deployment" yaml:"deployment" required:"true" description:"Deployment configuration for the platform"` + Patches []v1alpha1.PatchEntry `json:"patches,omitempty" yaml:"patches,omitempty" description:"Patch operations to apply to generated materials"` + Infrastructure Infrastructure `json:"infrastructure,omitzero" yaml:"infrastructure,omitzero" description:"Infrastructure configuration for generating infrastructure manifests (e.g., Terraform)."` + Signoz SigNoz `json:"signoz,omitzero" yaml:"signoz,omitempty" description:"The configuration for the SigNoz molding"` + TelemetryStore TelemetryStore `json:"telemetrystore,omitzero" yaml:"telemetrystore,omitempty" description:"The configuration for the telemetry store molding"` + TelemetryKeeper TelemetryKeeper `json:"telemetrykeeper,omitzero" yaml:"telemetrykeeper,omitempty" description:"The configuration for the telemetry keeper molding"` + MetaStore MetaStore `json:"metastore,omitzero" yaml:"metastore,omitempty" description:"The configuration for the meta store molding"` + Ingester Ingester `json:"ingester,omitzero" yaml:"ingester,omitempty" description:"The configuration for the ingester molding"` + _ struct{} `additionalProperties:"false"` +} + +var _ v1alpha1.Machinery = (*Casting)(nil) + +// Default returns an Installation with every molding initialised from its +// default. +func Default() *Casting { + return &Casting{ + CastingMeta: v1alpha1.CastingMeta{ + TypeVersion: v1alpha1.TypeVersion{APIVersion: "v1alpha1"}, + Kind: v1alpha1.KindInstallation, + Metadata: v1alpha1.TypeMetadata{Name: "signoz"}, + }, + Spec: Spec{ + Infrastructure: DefaultInfrastructure(), + Signoz: DefaultSigNoz(), + TelemetryStore: DefaultTelemetryStore(), + TelemetryKeeper: DefaultTelemetryKeeper(), + MetaStore: DefaultMetaStore(), + Ingester: DefaultIngester(), + }, + } +} + +// Example returns a minimal Installation; the forge pipeline fills in defaults. +func Example() *Casting { + return &Casting{ + CastingMeta: v1alpha1.CastingMeta{ + TypeVersion: v1alpha1.TypeVersion{APIVersion: "v1alpha1"}, + Kind: v1alpha1.KindInstallation, + Metadata: v1alpha1.TypeMetadata{Name: "signoz"}, + }, + } +} + +// Kind reports the casting kind. Shadows the embedded CastingMeta.Kind field; +// the field stays reachable as c.CastingMeta.Kind. +func (c *Casting) Kind() v1alpha1.Kind { + return v1alpha1.KindInstallation +} + +// MergeStatusIntoSpec folds each molding's Status into its own Spec. +func (c *Casting) MergeStatusIntoSpec() error { + if err := c.Spec.Signoz.Spec.MergeStatus(c.Spec.Signoz.Status.MoldingStatus); err != nil { + return err + } + if err := c.Spec.TelemetryStore.Spec.MergeStatus(c.Spec.TelemetryStore.Status.MoldingStatus); err != nil { + return err + } + if err := c.Spec.TelemetryKeeper.Spec.MergeStatus(c.Spec.TelemetryKeeper.Status.MoldingStatus); err != nil { + return err + } + if err := c.Spec.MetaStore.Spec.MergeStatus(c.Spec.MetaStore.Status.MoldingStatus); err != nil { + return err + } + if err := c.Spec.Ingester.Spec.MergeStatus(c.Spec.Ingester.Status.MoldingStatus); err != nil { + return err + } + return nil +} + +// TrackableProperties returns analytics tags for the casting. +func (c *Casting) TrackableProperties() domain.Properties { + return domain.NewProperties(). + Set("kind", v1alpha1.KindInstallation.String()). + Set("platform", c.Spec.Deployment.Platform.String()). + Set("mode", c.Spec.Deployment.Mode.String()). + Set("flavor", c.Spec.Deployment.Flavor.String()). + Set("patches_count", len(c.Spec.Patches)). + Set("infrastructure_enabled", c.Spec.Infrastructure.Enabled). + Set("metastore_kind", c.Spec.MetaStore.Kind.String()). + Set("telemetrystore_kind", c.Spec.TelemetryStore.Kind.String()). + Set("telemetrykeeper_kind", c.Spec.TelemetryKeeper.Kind.String()) +} diff --git a/api/v1alpha1/installation/casting.schema.json b/api/v1alpha1/installation/casting.schema.json new file mode 100644 index 0000000..31c7451 --- /dev/null +++ b/api/v1alpha1/installation/casting.schema.json @@ -0,0 +1,710 @@ +{ + "required": [ + "apiVersion", + "kind", + "metadata", + "spec" + ], + "additionalProperties": false, + "definitions": { + "InstallationInfrastructure": { + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "status": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + } + }, + "type": "object" + }, + "InstallationIngester": { + "additionalProperties": false, + "properties": { + "spec": { + "$ref": "#/definitions/V1Alpha1MoldingSpec" + }, + "status": { + "$ref": "#/definitions/InstallationIngesterStatus" + } + }, + "type": "object" + }, + "InstallationIngesterStatus": { + "additionalProperties": false, + "properties": { + "addresses": { + "$ref": "#/definitions/InstallationIngesterStatusAddresses" + }, + "config": { + "$ref": "#/definitions/V1Alpha1TypeConfig", + "description": "Configuration for the molding" + }, + "env": { + "description": "Environment variables for the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "extras": { + "description": "Extra information about the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + } + }, + "type": "object" + }, + "InstallationIngesterStatusAddresses": { + "additionalProperties": false, + "properties": { + "otlp": { + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" + }, + "InstallationMetaStore": { + "additionalProperties": false, + "properties": { + "kind": { + "$ref": "#/definitions/InstallationMetaStoreKind", + "description": "Kind of the meta store to use", + "examples": [ + "postgres", + "sqlite" + ] + }, + "spec": { + "$ref": "#/definitions/V1Alpha1MoldingSpec", + "description": "Specification for the meta store" + }, + "status": { + "$ref": "#/definitions/InstallationMetaStoreStatus", + "description": "Status of the meta store" + } + }, + "type": "object" + }, + "InstallationMetaStoreKind": { + "enum": [ + "postgres", + "sqlite" + ], + "type": "string" + }, + "InstallationMetaStoreStatus": { + "additionalProperties": false, + "properties": { + "addresses": { + "$ref": "#/definitions/InstallationMetaStoreStatusAddresses", + "description": "Addresses of the meta store" + }, + "config": { + "$ref": "#/definitions/V1Alpha1TypeConfig", + "description": "Configuration for the molding" + }, + "env": { + "description": "Environment variables for the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "extras": { + "description": "Extra information about the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + } + }, + "type": "object" + }, + "InstallationMetaStoreStatusAddresses": { + "additionalProperties": false, + "properties": { + "dsn": { + "description": "DSN addresses", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" + }, + "InstallationSigNoz": { + "additionalProperties": false, + "properties": { + "spec": { + "$ref": "#/definitions/V1Alpha1MoldingSpec" + }, + "status": { + "$ref": "#/definitions/InstallationSigNozStatus" + } + }, + "type": "object" + }, + "InstallationSigNozStatus": { + "additionalProperties": false, + "properties": { + "addresses": { + "$ref": "#/definitions/InstallationSigNozStatusAddresses" + }, + "config": { + "$ref": "#/definitions/V1Alpha1TypeConfig", + "description": "Configuration for the molding" + }, + "env": { + "description": "Environment variables for the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "extras": { + "description": "Extra information about the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + } + }, + "type": "object" + }, + "InstallationSigNozStatusAddresses": { + "additionalProperties": false, + "properties": { + "apiserver": { + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "opamp": { + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" + }, + "InstallationSpec": { + "required": [ + "deployment" + ], + "additionalProperties": false, + "properties": { + "deployment": { + "$ref": "#/definitions/V1Alpha1TypeDeployment", + "description": "Deployment configuration for the platform" + }, + "infrastructure": { + "$ref": "#/definitions/InstallationInfrastructure", + "description": "Infrastructure configuration for generating infrastructure manifests (e.g., Terraform)." + }, + "ingester": { + "$ref": "#/definitions/InstallationIngester", + "description": "The configuration for the ingester molding" + }, + "metastore": { + "$ref": "#/definitions/InstallationMetaStore", + "description": "The configuration for the meta store molding" + }, + "patches": { + "description": "Patch operations to apply to generated materials", + "items": { + "$ref": "#/definitions/V1Alpha1PatchEntry" + }, + "type": "array" + }, + "signoz": { + "$ref": "#/definitions/InstallationSigNoz", + "description": "The configuration for the SigNoz molding" + }, + "telemetrykeeper": { + "$ref": "#/definitions/InstallationTelemetryKeeper", + "description": "The configuration for the telemetry keeper molding" + }, + "telemetrystore": { + "$ref": "#/definitions/InstallationTelemetryStore", + "description": "The configuration for the telemetry store molding" + } + }, + "type": "object" + }, + "InstallationTelemetryKeeper": { + "additionalProperties": false, + "properties": { + "kind": { + "$ref": "#/definitions/InstallationTelemetryKeeperKind", + "description": "Kind of the telemetry keeper to use", + "examples": [ + "clickhousekeeper" + ] + }, + "spec": { + "$ref": "#/definitions/V1Alpha1MoldingSpec", + "description": "Specification for the telemetry keeper" + }, + "status": { + "$ref": "#/definitions/InstallationTelemetryKeeperStatus", + "description": "Status of the telemetry keeper" + } + }, + "type": "object" + }, + "InstallationTelemetryKeeperKind": { + "enum": [ + "clickhousekeeper" + ], + "type": "string" + }, + "InstallationTelemetryKeeperStatus": { + "additionalProperties": false, + "properties": { + "addresses": { + "$ref": "#/definitions/InstallationTelemetryKeeperStatusAddresses", + "description": "Addresses of the telemetry keeper" + }, + "config": { + "$ref": "#/definitions/V1Alpha1TypeConfig", + "description": "Configuration for the molding" + }, + "env": { + "description": "Environment variables for the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "extras": { + "description": "Extra information about the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + } + }, + "type": "object" + }, + "InstallationTelemetryKeeperStatusAddresses": { + "additionalProperties": false, + "properties": { + "client": { + "description": "Client addresses", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "raft": { + "description": "Raft addresses", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" + }, + "InstallationTelemetryStore": { + "additionalProperties": false, + "properties": { + "kind": { + "$ref": "#/definitions/InstallationTelemetryStoreKind", + "description": "Kind of the telemetry store to use", + "examples": [ + "clickhouse" + ] + }, + "spec": { + "$ref": "#/definitions/V1Alpha1MoldingSpec", + "description": "Specification for the telemetry store" + }, + "status": { + "$ref": "#/definitions/InstallationTelemetryStoreStatus", + "description": "Status of the telemetry store" + } + }, + "type": "object" + }, + "InstallationTelemetryStoreKind": { + "enum": [ + "clickhouse" + ], + "type": "string" + }, + "InstallationTelemetryStoreStatus": { + "additionalProperties": false, + "properties": { + "addresses": { + "$ref": "#/definitions/InstallationTelemetryStoreStatusAddresses", + "description": "Addresses of the telemetry store" + }, + "config": { + "$ref": "#/definitions/V1Alpha1TypeConfig", + "description": "Configuration for the molding" + }, + "env": { + "description": "Environment variables for the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "extras": { + "description": "Extra information about the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + } + }, + "type": "object" + }, + "InstallationTelemetryStoreStatusAddresses": { + "additionalProperties": false, + "properties": { + "tcp": { + "description": "TCP addresses", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" + }, + "V1Alpha1Flavor": { + "enum": [ + "compose", + "swarm", + "binary", + "kustomize", + "helm", + "blueprint", + "stack", + "template", + "terraform" + ], + "type": "string" + }, + "V1Alpha1Kind": { + "enum": [ + "Installation" + ], + "type": "string" + }, + "V1Alpha1Mode": { + "enum": [ + "docker", + "systemd", + "kubernetes", + "ec2" + ], + "type": "string" + }, + "V1Alpha1MoldingSpec": { + "additionalProperties": false, + "properties": { + "cluster": { + "$ref": "#/definitions/V1Alpha1TypeCluster", + "description": "Cluster configuration for the molding" + }, + "config": { + "$ref": "#/definitions/V1Alpha1TypeConfig", + "description": "Configuration for the molding" + }, + "enabled": { + "description": "Whether the molding is enabled", + "default": true, + "type": [ + "null", + "boolean" + ] + }, + "env": { + "description": "Environment variables for the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "image": { + "description": "Container image of the molding", + "examples": [ + "signoz/signoz:latest" + ], + "pattern": "^[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*(:[a-zA-Z0-9._-]+)?(@sha256:[a-f0-9]{64})?$", + "type": "string" + }, + "version": { + "description": "The version of the molding to use", + "examples": [ + "latest" + ], + "type": "string" + } + }, + "type": "object" + }, + "V1Alpha1PatchEntry": { + "required": [ + "target", + "operations" + ], + "additionalProperties": false, + "properties": { + "operations": { + "description": "JSON Patch (RFC 6902) operations to apply. Used by the jsonpatch driver.", + "items": { + "$ref": "#/definitions/V1Alpha1PatchOperation" + }, + "minItems": 1, + "type": [ + "array", + "null" + ] + }, + "target": { + "description": "Target output file to patch", + "examples": [ + "compose.yaml", + "signoz/deployment.yaml", + "values.yaml", + "telemetrystore/telemtrystore-clickhouse-0-*.yaml" + ], + "minLength": 1, + "type": "string" + }, + "type": { + "description": "Patch driver type. Defaults to jsonpatch.", + "default": "jsonpatch", + "examples": [ + "jsonpatch" + ], + "enum": [ + "", + "jsonpatch" + ], + "type": "string" + } + }, + "type": "object" + }, + "V1Alpha1PatchOperation": { + "required": [ + "op", + "path" + ], + "additionalProperties": false, + "properties": { + "from": { + "description": "Source JSON Pointer for move and copy operations", + "examples": [ + "/services/clickhouse/old_field" + ], + "pattern": "^/", + "type": "string" + }, + "op": { + "description": "JSON Patch (RFC 6902) operation type", + "enum": [ + "add", + "remove", + "replace", + "move", + "copy", + "test" + ], + "type": "string" + }, + "path": { + "description": "JSON Pointer (RFC 6901) to the target location", + "examples": [ + "/services/clickhouse/mem_limit" + ], + "pattern": "^/", + "type": "string" + }, + "value": { + "description": "Value for add, replace, or test operations" + } + }, + "type": "object" + }, + "V1Alpha1Platform": { + "enum": [ + "render", + "coolify", + "railway", + "ecs", + "aws", + "gcp", + "azure" + ], + "type": "string" + }, + "V1Alpha1Status": { + "additionalProperties": false, + "properties": { + "checksum": { + "description": "Checksum of the casting file", + "type": "string" + } + }, + "type": "object" + }, + "V1Alpha1TypeCluster": { + "additionalProperties": false, + "properties": { + "replicas": { + "description": "Number of replicas for the molding.", + "examples": [ + 1 + ], + "minimum": 0, + "type": [ + "null", + "integer" + ] + }, + "shards": { + "description": "Number of shards for the molding", + "examples": [ + 1 + ], + "minimum": 1, + "type": [ + "null", + "integer" + ] + } + }, + "type": "object" + }, + "V1Alpha1TypeConfig": { + "additionalProperties": false, + "properties": { + "data": { + "description": "Configuration data as key-value pairs.", + "additionalProperties": { + "type": "string" + }, + "type": "object" + } + }, + "type": "object" + }, + "V1Alpha1TypeDeployment": { + "additionalProperties": false, + "properties": { + "flavor": { + "$ref": "#/definitions/V1Alpha1Flavor", + "description": "Flavor of mode for the deployment" + }, + "mode": { + "$ref": "#/definitions/V1Alpha1Mode", + "description": "Type of installation method" + }, + "platform": { + "$ref": "#/definitions/V1Alpha1Platform", + "description": "Provider where an installation runs on" + } + }, + "type": "object" + }, + "V1Alpha1TypeMetadata": { + "required": [ + "name" + ], + "additionalProperties": false, + "properties": { + "annotations": { + "description": "Annotations is an unstructured key-value map for arbitrary metadata. Can be used to specify deployment-specific settings.", + "additionalProperties": { + "type": "string" + }, + "type": [ + "object", + "null" + ] + }, + "name": { + "description": "The name of this installation. This name is used to identify the installation.", + "default": "signoz", + "examples": [ + "signoz" + ], + "maxLength": 63, + "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", + "type": "string" + } + }, + "type": "object" + } + }, + "properties": { + "apiVersion": { + "description": "API Version of the configuration schema.", + "default": "v1alpha1", + "examples": [ + "v1alpha1" + ], + "enum": [ + "v1alpha1" + ], + "type": "string" + }, + "kind": { + "$ref": "#/definitions/V1Alpha1Kind", + "description": "Kind of the casting resource." + }, + "metadata": { + "$ref": "#/definitions/V1Alpha1TypeMetadata", + "description": "Metadata of the casting configuration" + }, + "spec": { + "$ref": "#/definitions/InstallationSpec", + "description": "Installation specification" + }, + "status": { + "$ref": "#/definitions/V1Alpha1Status", + "description": "Status of the casting" + } + }, + "type": "object" +} \ No newline at end of file diff --git a/api/v1alpha1/installation/infrastructure.go b/api/v1alpha1/installation/infrastructure.go new file mode 100644 index 0000000..7baa07a --- /dev/null +++ b/api/v1alpha1/installation/infrastructure.go @@ -0,0 +1,36 @@ +package installation + +import "encoding/json" + +// Infrastructure holds the configuration for infrastructure manifest generation (e.g., Terraform). +// The cloud provider is resolved automatically from spec.deployment.platform — no provider field +// is needed here. +type Infrastructure struct { + // Whether infrastructure manifest generation is enabled + Enabled bool `json:"enabled" yaml:"enabled"` + + // Status holds the generated IaC file contents keyed by filename (e.g. "main.tf.json"). + // This is populated by foundry after generation and written to the lock file. + Status map[string]string `json:"status,omitempty" yaml:"status,omitempty"` + + _ struct{} `additionalProperties:"false"` +} + +// MarshalJSON implements json.Marshaler. It manually omits Status when zero +// so that the strategic merge patch doesn't overwrite defaults with empty values. +func (i Infrastructure) MarshalJSON() ([]byte, error) { + m := map[string]any{ + "enabled": i.Enabled, + } + if len(i.Status) > 0 { + m["status"] = i.Status + } + return json.Marshal(m) +} + +// DefaultInfrastructure returns the default Infrastructure configuration. +func DefaultInfrastructure() Infrastructure { + return Infrastructure{ + Enabled: false, + } +} diff --git a/api/v1alpha1/ingester.go b/api/v1alpha1/installation/ingester.go similarity index 61% rename from api/v1alpha1/ingester.go rename to api/v1alpha1/installation/ingester.go index d0e1f2c..3d217c2 100644 --- a/api/v1alpha1/ingester.go +++ b/api/v1alpha1/installation/ingester.go @@ -1,36 +1,42 @@ -package v1alpha1 +package installation import "github.com/hanzoai/o11y-foundry/internal/types" type Ingester struct { // Specification for the ingester. - Spec MoldingSpec `json:"spec" yaml:"spec" jsonschema:"description=Specification for the ingester"` + Spec v1alpha1.MoldingSpec `json:"spec" yaml:"spec" jsonschema:"description=Specification for the ingester"` // Status of the ingester. Status IngesterStatus `json:"status" yaml:"status,omitempty" jsonschema:"description=Status of the ingester"` + + _ struct{} `additionalProperties:"false"` } type IngesterStatus struct { - MoldingStatus `json:",inline" yaml:",inline"` + v1alpha1.MoldingStatus `json:",inline" yaml:",inline"` Addresses IngesterStatusAddresses `json:"addresses" yaml:"addresses,omitempty" jsonschema:"description=Addresses of the ingester"` + + _ struct{} `additionalProperties:"false"` } type IngesterStatusAddresses struct { OTLP []string `json:"otlp" yaml:"otlp" jsonschema:"description=OTLP addresses"` + + _ struct{} `additionalProperties:"false"` } func DefaultIngester() Ingester { return Ingester{ - Spec: MoldingSpec{ - Enabled: true, - Cluster: TypeCluster{ - Replicas: types.NewIntPtr(1), + Spec: v1alpha1.MoldingSpec{ + Enabled: v1alpha1.BoolPtr(true), + Cluster: v1alpha1.TypeCluster{ + Replicas: v1alpha1.IntPtr(1), }, Version: "latest", Image: "ghcr.io/hanzoai/otel-collector:latest", Env: map[string]string{}, - Config: TypeConfig{ + Config: v1alpha1.TypeConfig{ Data: map[string]string{}, }, }, diff --git a/api/v1alpha1/metastore.go b/api/v1alpha1/installation/metastore.go similarity index 66% rename from api/v1alpha1/metastore.go rename to api/v1alpha1/installation/metastore.go index 235e693..d030947 100644 --- a/api/v1alpha1/metastore.go +++ b/api/v1alpha1/installation/metastore.go @@ -1,4 +1,4 @@ -package v1alpha1 +package installation import ( "github.com/hanzoai/o11y-foundry/internal/types" @@ -9,30 +9,35 @@ type MetaStore struct { Kind MetaStoreKind `json:"kind,omitzero" yaml:"kind,omitempty" description:"Kind of the meta store to use" examples:"[\"postgres\",\"sqlite\"]"` // Specification for the meta store. - Spec MoldingSpec `json:"spec" yaml:"spec" description:"Specification for the meta store"` + Spec v1alpha1.MoldingSpec `json:"spec" yaml:"spec" description:"Specification for the meta store"` // Status of the meta store. Status MetaStoreStatus `json:"status" yaml:"status,omitempty" description:"Status of the meta store"` + + _ struct{} `additionalProperties:"false"` } type MetaStoreStatus struct { - MoldingStatus `json:",inline" yaml:",inline"` + v1alpha1.MoldingStatus `json:",inline" yaml:",inline"` Addresses MetaStoreStatusAddresses `json:"addresses" yaml:"addresses,omitempty" description:"Addresses of the meta store"` + + _ struct{} `additionalProperties:"false"` } type MetaStoreStatusAddresses struct { // DSN addresses. DSN []string `json:"dsn" yaml:"dsn" description:"DSN addresses"` + _ struct{} `additionalProperties:"false"` } func DefaultMetaStore() MetaStore { return MetaStore{ Kind: MetaStoreKindPostgres, - Spec: MoldingSpec{ - Enabled: true, - Cluster: TypeCluster{ - Replicas: types.NewIntPtr(1), + Spec: v1alpha1.MoldingSpec{ + Enabled: v1alpha1.BoolPtr(true), + Cluster: v1alpha1.TypeCluster{ + Replicas: v1alpha1.IntPtr(1), }, Version: "16", Image: "postgres:16", diff --git a/api/v1alpha1/metastore_kind.go b/api/v1alpha1/installation/metastore_kind.go similarity index 85% rename from api/v1alpha1/metastore_kind.go rename to api/v1alpha1/installation/metastore_kind.go index de822b1..4689a6e 100644 --- a/api/v1alpha1/metastore_kind.go +++ b/api/v1alpha1/installation/metastore_kind.go @@ -1,10 +1,11 @@ -package v1alpha1 +package installation import ( "encoding/json" "errors" "fmt" + "github.com/swaggest/jsonschema-go" "go.yaml.in/yaml/v3" ) @@ -13,6 +14,7 @@ var _ yaml.Unmarshaler = (*MetaStoreKind)(nil) var _ json.Marshaler = (*MetaStoreKind)(nil) var _ json.Unmarshaler = (*MetaStoreKind)(nil) var _ fmt.Stringer = (*MetaStoreKind)(nil) +var _ jsonschema.Enum = (*MetaStoreKind)(nil) var ( MetaStoreKindPostgres MetaStoreKind = MetaStoreKind{s: "postgres"} @@ -69,3 +71,12 @@ func (kind *MetaStoreKind) UnmarshalYAML(node *yaml.Node) error { func (kind MetaStoreKind) MarshalYAML() (any, error) { return kind.String(), nil } + +func (kind MetaStoreKind) Enum() []any { + kinds := []any{} + for _, kind := range MetaStoreKinds() { + kinds = append(kinds, kind.String()) + } + + return kinds +} diff --git a/api/v1alpha1/installation/schema.go b/api/v1alpha1/installation/schema.go new file mode 100644 index 0000000..52af859 --- /dev/null +++ b/api/v1alpha1/installation/schema.go @@ -0,0 +1,18 @@ +package installation + +import ( + _ "embed" + + "github.com/google/jsonschema-go/jsonschema" + "github.com/signoz/foundry/api/v1alpha1" +) + +//go:embed casting.schema.json +var schemaJSON []byte + +var schema = v1alpha1.MustResolveSchema(schemaJSON) + +// Schema returns the resolved JSON schema for an Installation casting. +func Schema() *jsonschema.Resolved { + return schema +} diff --git a/api/v1alpha1/installation/schema_test.go b/api/v1alpha1/installation/schema_test.go new file mode 100644 index 0000000..b53fac8 --- /dev/null +++ b/api/v1alpha1/installation/schema_test.go @@ -0,0 +1,26 @@ +package installation + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSchema(t *testing.T) { + t.Parallel() + assert.NotNil(t, Schema()) +} + +func TestSchemaValidatesDefault(t *testing.T) { + t.Parallel() + + contents, err := json.Marshal(Default()) + require.NoError(t, err) + + var payload map[string]any + require.NoError(t, json.Unmarshal(contents, &payload)) + + assert.NoError(t, Schema().Validate(payload)) +} diff --git a/api/v1alpha1/installation/signoz.go b/api/v1alpha1/installation/signoz.go new file mode 100644 index 0000000..15ec7a8 --- /dev/null +++ b/api/v1alpha1/installation/signoz.go @@ -0,0 +1,44 @@ +package installation + +import "github.com/signoz/foundry/api/v1alpha1" + +type SigNoz struct { + // Specification for signoz. + Spec v1alpha1.MoldingSpec `json:"spec" yaml:"spec" jsonschema:"description=Specification for SigNoz"` + + // Status of signoz. + Status SigNozStatus `json:"status" yaml:"status,omitempty" jsonschema:"description=Status of SigNoz"` + + _ struct{} `additionalProperties:"false"` +} + +type SigNozStatus struct { + v1alpha1.MoldingStatus `json:",inline" yaml:",inline"` + + Addresses SigNozStatusAddresses `json:"addresses" yaml:"addresses,omitempty" jsonschema:"description=Addresses of SigNoz"` + + _ struct{} `additionalProperties:"false"` +} + +type SigNozStatusAddresses struct { + // API server addresses. + APIServer []string `json:"apiserver" yaml:"apiserver" jsonschema:"description=API server addresses"` + + // Opamp server addresses. + Opamp []string `json:"opamp" yaml:"opamp" jsonschema:"description=Opamp server addresses"` + + _ struct{} `additionalProperties:"false"` +} + +func DefaultSigNoz() SigNoz { + return SigNoz{ + Spec: v1alpha1.MoldingSpec{ + Enabled: v1alpha1.BoolPtr(true), + Cluster: v1alpha1.TypeCluster{ + Replicas: v1alpha1.IntPtr(1), + }, + Version: "latest", + Image: "signoz/signoz:latest", + }, + } +} diff --git a/api/v1alpha1/telemetrykeeper.go b/api/v1alpha1/installation/telemetrykeeper.go similarity index 72% rename from api/v1alpha1/telemetrykeeper.go rename to api/v1alpha1/installation/telemetrykeeper.go index edb614c..9a6e15f 100644 --- a/api/v1alpha1/telemetrykeeper.go +++ b/api/v1alpha1/installation/telemetrykeeper.go @@ -1,4 +1,4 @@ -package v1alpha1 +package installation import ( "github.com/hanzoai/o11y-foundry/internal/types" @@ -9,17 +9,21 @@ type TelemetryKeeper struct { Kind TelemetryKeeperKind `json:"kind,omitzero" yaml:"kind,omitempty" description:"Kind of the telemetry keeper to use" examples:"[\"clickhousekeeper\"]"` // Specification for the telemetry keeper. - Spec MoldingSpec `json:"spec" yaml:"spec" description:"Specification for the telemetry keeper"` + Spec v1alpha1.MoldingSpec `json:"spec" yaml:"spec" description:"Specification for the telemetry keeper"` // Status of the telemetry keeper. Status TelemetryKeeperStatus `json:"status" yaml:"status,omitempty" description:"Status of the telemetry keeper"` + + _ struct{} `additionalProperties:"false"` } type TelemetryKeeperStatus struct { - MoldingStatus `json:",inline" yaml:",inline"` + v1alpha1.MoldingStatus `json:",inline" yaml:",inline"` // Addresses of the telemetry keeper. Addresses TelemetryKeeperStatusAddresses `json:"addresses" yaml:"addresses,omitempty" description:"Addresses of the telemetry keeper"` + + _ struct{} `additionalProperties:"false"` } type TelemetryKeeperStatusAddresses struct { @@ -28,15 +32,17 @@ type TelemetryKeeperStatusAddresses struct { // Client addresses. Client []string `json:"client" yaml:"client,omitempty" description:"Client addresses"` + + _ struct{} `additionalProperties:"false"` } func DefaultTelemetryKeeper() TelemetryKeeper { return TelemetryKeeper{ Kind: TelemetryKeeperKindClickhouseKeeper, - Spec: MoldingSpec{ - Enabled: true, - Cluster: TypeCluster{ - Replicas: types.NewIntPtr(1), + Spec: v1alpha1.MoldingSpec{ + Enabled: v1alpha1.BoolPtr(true), + Cluster: v1alpha1.TypeCluster{ + Replicas: v1alpha1.IntPtr(1), }, Version: "25.5.6", Image: "clickhouse/clickhouse-keeper:25.5.6", diff --git a/api/v1alpha1/telemetrykeeper_kind.go b/api/v1alpha1/installation/telemetrykeeper_kind.go similarity index 85% rename from api/v1alpha1/telemetrykeeper_kind.go rename to api/v1alpha1/installation/telemetrykeeper_kind.go index 092e146..84ec5db 100644 --- a/api/v1alpha1/telemetrykeeper_kind.go +++ b/api/v1alpha1/installation/telemetrykeeper_kind.go @@ -1,10 +1,11 @@ -package v1alpha1 +package installation import ( "encoding/json" "errors" "fmt" + "github.com/swaggest/jsonschema-go" "go.yaml.in/yaml/v3" ) @@ -13,6 +14,7 @@ var _ yaml.Unmarshaler = (*TelemetryKeeperKind)(nil) var _ json.Marshaler = (*TelemetryKeeperKind)(nil) var _ json.Unmarshaler = (*TelemetryKeeperKind)(nil) var _ fmt.Stringer = (*TelemetryKeeperKind)(nil) +var _ jsonschema.Enum = (*TelemetryKeeperKind)(nil) var ( TelemetryKeeperKindClickhouseKeeper TelemetryKeeperKind = TelemetryKeeperKind{s: "clickhousekeeper"} @@ -68,3 +70,12 @@ func (kind *TelemetryKeeperKind) UnmarshalYAML(node *yaml.Node) error { func (kind TelemetryKeeperKind) MarshalYAML() (any, error) { return kind.String(), nil } + +func (kind TelemetryKeeperKind) Enum() []any { + kinds := []any{} + for _, kind := range TelemetryKeeperKinds() { + kinds = append(kinds, kind.String()) + } + + return kinds +} diff --git a/api/v1alpha1/telemetrystore.go b/api/v1alpha1/installation/telemetrystore.go similarity index 66% rename from api/v1alpha1/telemetrystore.go rename to api/v1alpha1/installation/telemetrystore.go index 99f800d..2ba1275 100644 --- a/api/v1alpha1/telemetrystore.go +++ b/api/v1alpha1/installation/telemetrystore.go @@ -1,4 +1,4 @@ -package v1alpha1 +package installation import ( "github.com/hanzoai/o11y-foundry/internal/types" @@ -9,31 +9,37 @@ type TelemetryStore struct { Kind TelemetryStoreKind `json:"kind,omitzero" yaml:"kind,omitempty" description:"Kind of the telemetry store to use" examples:"[\"clickhouse\"]"` // Specification for the telemetry store. - Spec MoldingSpec `json:"spec" yaml:"spec" description:"Specification for the telemetry store"` + Spec v1alpha1.MoldingSpec `json:"spec" yaml:"spec" description:"Specification for the telemetry store"` // Status of the telemetry store. Status TelemetryStoreStatus `json:"status" yaml:"status,omitempty" description:"Status of the telemetry store"` + + _ struct{} `additionalProperties:"false"` } type TelemetryStoreStatus struct { - MoldingStatus `json:",inline" yaml:",inline"` + v1alpha1.MoldingStatus `json:",inline" yaml:",inline"` Addresses TelemetryStoreStatusAddresses `json:"addresses" yaml:"addresses,omitempty" description:"Addresses of the telemetry store"` + + _ struct{} `additionalProperties:"false"` } type TelemetryStoreStatusAddresses struct { // TCP addresses. TCP []string `json:"tcp" yaml:"tcp" description:"TCP addresses"` + + _ struct{} `additionalProperties:"false"` } func DefaultTelemetryStore() TelemetryStore { return TelemetryStore{ Kind: TelemetryStoreKindClickhouse, - Spec: MoldingSpec{ - Enabled: true, - Cluster: TypeCluster{ - Replicas: types.NewIntPtr(0), - Shards: types.NewIntPtr(1), + Spec: v1alpha1.MoldingSpec{ + Enabled: v1alpha1.BoolPtr(true), + Cluster: v1alpha1.TypeCluster{ + Replicas: v1alpha1.IntPtr(0), + Shards: v1alpha1.IntPtr(1), }, Version: "25.5.6", Image: "ghcr.io/hanzoai/datastore:25.5.6", diff --git a/api/v1alpha1/telemetrystore_kind.go b/api/v1alpha1/installation/telemetrystore_kind.go similarity index 85% rename from api/v1alpha1/telemetrystore_kind.go rename to api/v1alpha1/installation/telemetrystore_kind.go index 742171c..2dfc650 100644 --- a/api/v1alpha1/telemetrystore_kind.go +++ b/api/v1alpha1/installation/telemetrystore_kind.go @@ -1,10 +1,11 @@ -package v1alpha1 +package installation import ( "encoding/json" "errors" "fmt" + "github.com/swaggest/jsonschema-go" "go.yaml.in/yaml/v3" ) @@ -13,6 +14,7 @@ var _ yaml.Unmarshaler = (*TelemetryStoreKind)(nil) var _ json.Marshaler = (*TelemetryStoreKind)(nil) var _ json.Unmarshaler = (*TelemetryStoreKind)(nil) var _ fmt.Stringer = (*TelemetryStoreKind)(nil) +var _ jsonschema.Enum = (*TelemetryStoreKind)(nil) var ( TelemetryStoreKindClickhouse TelemetryStoreKind = TelemetryStoreKind{s: "clickhouse"} @@ -50,10 +52,12 @@ func (kind *TelemetryStoreKind) UnmarshalText(text []byte) error { return nil } } + if text == nil { *kind = TelemetryStoreKind{s: ""} return nil } + return errors.New("invalid telemetry store kind: " + string(text)) } @@ -68,3 +72,12 @@ func (kind *TelemetryStoreKind) UnmarshalYAML(node *yaml.Node) error { func (kind TelemetryStoreKind) MarshalYAML() (any, error) { return kind.String(), nil } + +func (kind TelemetryStoreKind) Enum() []any { + kinds := []any{} + for _, kind := range TelemetryStoreKinds() { + kinds = append(kinds, kind.String()) + } + + return kinds +} diff --git a/api/v1alpha1/machinery.go b/api/v1alpha1/machinery.go new file mode 100644 index 0000000..302f64f --- /dev/null +++ b/api/v1alpha1/machinery.go @@ -0,0 +1,9 @@ +package v1alpha1 + +import "github.com/signoz/foundry/internal/domain" + +// Machinery is the marker every per-Kind casting type satisfies. +type Machinery interface { + Kind() Kind + TrackableProperties() domain.Properties +} diff --git a/api/v1alpha1/merge.go b/api/v1alpha1/merge.go new file mode 100644 index 0000000..26c1d96 --- /dev/null +++ b/api/v1alpha1/merge.go @@ -0,0 +1,59 @@ +package v1alpha1 + +import ( + "encoding/json" + "reflect" + + "github.com/signoz/foundry/internal/errors" + "k8s.io/apimachinery/pkg/util/strategicpatch" +) + +// Merge applies an override onto a base via Kubernetes strategic merge patch +// and mutates base in place. Both arguments must be pointers to the same +// concrete struct type. Fields typed as any are replaced wholesale rather than +// recursively merged. +func Merge(base, overrides any) error { + if overrides == nil { + return nil + } + + baseBytes, err := json.Marshal(base) + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to convert current object to byte sequence") + } + + overrideBytes, err := json.Marshal(overrides) + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to convert current object to byte sequence") + } + + patchMeta, err := strategicpatch.NewPatchMetaFromStruct(base) + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to produce patch meta from struct") + } + + patch, err := strategicpatch.CreateThreeWayMergePatch(overrideBytes, overrideBytes, baseBytes, patchMeta, true) + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to create three way merge patch") + } + + merged, err := strategicpatch.StrategicMergePatchUsingLookupPatchMeta(baseBytes, patch, patchMeta) + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to apply patch") + } + + valueOfBase := reflect.Indirect(reflect.ValueOf(base)) + + into := reflect.New(valueOfBase.Type()) + if err := json.Unmarshal(merged, into.Interface()); err != nil { + return err + } + + if !valueOfBase.CanSet() { + return errors.Newf(errors.TypeInternal, "unable to set unmarshalled value into base object") + } + + valueOfBase.Set(reflect.Indirect(into)) + + return nil +} diff --git a/api/v1alpha1/merge_test.go b/api/v1alpha1/merge_test.go new file mode 100644 index 0000000..bcb1de8 --- /dev/null +++ b/api/v1alpha1/merge_test.go @@ -0,0 +1,49 @@ +package v1alpha1 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMerge(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + base TypeMetadata + override TypeMetadata + want TypeMetadata + }{ + { + name: "override Name wins over base", + base: TypeMetadata{Name: "base"}, + override: TypeMetadata{Name: "override"}, + want: TypeMetadata{Name: "override"}, + }, + { + name: "override fills in unset base fields", + base: TypeMetadata{}, + override: TypeMetadata{Name: "fresh"}, + want: TypeMetadata{Name: "fresh"}, + }, + { + name: "override Annotations (map with omitempty) does not clobber base when unset", + base: TypeMetadata{Name: "base", Annotations: map[string]string{"a": "1"}}, + override: TypeMetadata{Name: "base"}, + want: TypeMetadata{Name: "base", Annotations: map[string]string{"a": "1"}}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + base := tc.base + override := tc.override + require.NoError(t, Merge(&base, &override)) + assert.Equal(t, tc.want, base) + }) + } +} diff --git a/api/v1alpha1/meta.go b/api/v1alpha1/meta.go index d1a4fc3..8c98e9c 100644 --- a/api/v1alpha1/meta.go +++ b/api/v1alpha1/meta.go @@ -1,16 +1,7 @@ package v1alpha1 -import ( - "encoding/json" - "fmt" - "reflect" - - "k8s.io/apimachinery/pkg/util/strategicpatch" -) - type TypeVersion struct { - // API Version of the casting configuration schema. - APIVersion string `json:"apiVersion,omitempty" yaml:"apiVersion,omitempty" description:"API Version of the casting configuration schema" example:"v1alpha1"` + APIVersion string `json:"apiVersion" yaml:"apiVersion" required:"true" nullable:"false" enum:"v1alpha1" description:"API Version of the configuration schema." default:"v1alpha1" example:"v1alpha1"` } type TypeMetadata struct { @@ -23,72 +14,25 @@ type TypeMetadata struct { } type TypeCluster struct { - // Number of replicas for the component - Replicas *int `json:"replicas,omitempty" yaml:"replicas,omitempty" description:"Number of replicas for the component" example:"1"` - - // Number of shards for the component - Shards *int `json:"shards,omitempty" yaml:"shards,omitempty" description:"Number of shards for the component" example:"1"` + Replicas *int `json:"replicas,omitempty" yaml:"replicas,omitempty" minimum:"0" description:"Number of replicas for the molding." example:"1"` + Shards *int `json:"shards,omitempty" yaml:"shards,omitempty" minimum:"1" description:"Number of shards for the molding" example:"1"` + _ struct{} `additionalProperties:"false"` } type TypeConfig struct { - // Data contains the configuration data. - Data map[string]string `json:"data,omitempty" yaml:"data,omitempty" description:"Configuration data as key-value pairs"` + Data map[string]string `json:"data,omitempty" yaml:"data,omitempty" description:"Configuration data as key-value pairs."` + _ struct{} `additionalProperties:"false"` } type TypeDeployment struct { - // Platform: Provider where an installation runs on using various cloud vendors - // Example values: aws|gcp|azure|digitalocean|railway - Platform string `json:"platform,omitempty" yaml:"platform,omitempty" description:"Provider where an installation runs on" examples:"[\"aws\",\"gcp\",\"azure\",\"digitalocean\",\"railway\",\"docker\",\"linux\"]"` + // Platform: cloud or hosting provider where an installation runs. + Platform Platform `json:"platform,omitzero" yaml:"platform,omitempty" description:"Provider where an installation runs on"` - // Mode: Type of installation method that we support, currently identifies the engine or technology behind a deployment - Mode string `json:"mode,omitempty" yaml:"mode,omitempty" description:"Type of installation method" examples:"[\"binary\",\"docker\",\"kubernetes\",\"helm\",\"nomad\",\"windows\",\"systemctl\"]"` + // Mode: type of installation method (engine or technology behind the deployment). + Mode Mode `json:"mode,omitzero" yaml:"mode,omitempty" description:"Type of installation method"` - // Flavor: Defines the flavor of mode for the deployment, allows the user the pattern to deploy on - Flavor string `json:"flavor,omitempty" yaml:"flavor,omitempty" description:"Flavor of mode for the deployment" examples:"[\"compose\",\"swarm\",\"helmfile\",\"helm\",\"kustomize\",\"binary\",\"rpm\",\"deb\",\"chocolatey\"]"` -} + // Flavor: variant of the mode for the deployment. + Flavor Flavor `json:"flavor,omitzero" yaml:"flavor,omitempty" description:"Flavor of mode for the deployment"` -func Merge(base, overrides any) error { - if overrides == nil { - return nil - } - - baseBytes, err := json.Marshal(base) - if err != nil { - return fmt.Errorf("failed to convert current object to byte sequence: %w", err) - } - - overrideBytes, err := json.Marshal(overrides) - if err != nil { - return fmt.Errorf("failed to convert current object to byte sequence: %w", err) - } - - patchMeta, err := strategicpatch.NewPatchMetaFromStruct(base) - if err != nil { - return fmt.Errorf("failed to produce patch meta from struct: %w", err) - } - - patch, err := strategicpatch.CreateThreeWayMergePatch(overrideBytes, overrideBytes, baseBytes, patchMeta, true) - if err != nil { - return fmt.Errorf("failed to create three way merge patch: %w", err) - } - - merged, err := strategicpatch.StrategicMergePatchUsingLookupPatchMeta(baseBytes, patch, patchMeta) - if err != nil { - return fmt.Errorf("failed to apply patch: %w", err) - } - - valueOfBase := reflect.Indirect(reflect.ValueOf(base)) - - into := reflect.New(valueOfBase.Type()) - if err := json.Unmarshal(merged, into.Interface()); err != nil { - return err - } - - if !valueOfBase.CanSet() { - return fmt.Errorf("unable to set unmarshalled value into base object") - } - - valueOfBase.Set(reflect.Indirect(into)) - - return nil + _ struct{} `additionalProperties:"false"` } diff --git a/api/v1alpha1/molding.go b/api/v1alpha1/molding.go index 2add3b7..cf4bf98 100644 --- a/api/v1alpha1/molding.go +++ b/api/v1alpha1/molding.go @@ -4,7 +4,7 @@ import "maps" type MoldingSpec struct { // Whether the molding is enabled - Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty" description:"Whether the molding is enabled" default:"true"` + Enabled *bool `json:"enabled,omitempty" yaml:"enabled,omitempty" description:"Whether the molding is enabled" default:"true"` // Cluster configuration for the molding Cluster TypeCluster `json:"cluster" yaml:"cluster,omitempty" description:"Cluster configuration for the molding"` @@ -20,6 +20,8 @@ type MoldingSpec struct { // Configuration for the molding Config TypeConfig `json:"config" yaml:"config,omitempty" description:"Configuration for the molding"` + + _ struct{} `additionalProperties:"false"` } type MoldingStatus struct { @@ -31,6 +33,17 @@ type MoldingStatus struct { // Configuration for the molding Config TypeConfig `json:"config" yaml:"config,omitempty" description:"Configuration for the molding"` + + _ struct{} `additionalProperties:"false"` +} + +// IsEnabled returns whether the molding is enabled. +// Returns false if the receiver or Enabled is nil. +func (spec *MoldingSpec) IsEnabled() bool { + if spec == nil || spec.Enabled == nil { + return false + } + return *spec.Enabled } func (spec *MoldingSpec) MergeStatus(status MoldingStatus) error { @@ -42,11 +55,14 @@ func (spec *MoldingSpec) MergeStatus(status MoldingStatus) error { status.Env = make(map[string]string) } - maps.Copy(spec.Env, status.Env) + maps.Copy(status.Env, spec.Env) + spec.Env = status.Env - if err := Merge(&spec.Config, status.Config); err != nil { + if err := Merge(&status.Config, spec.Config); err != nil { return err } + spec.Config = status.Config + return nil } diff --git a/api/v1alpha1/patch.go b/api/v1alpha1/patch.go new file mode 100644 index 0000000..5e157ca --- /dev/null +++ b/api/v1alpha1/patch.go @@ -0,0 +1,46 @@ +package v1alpha1 + +const ( + // PatchTypeJSONPatch is the default patch type using JSON Patch (RFC 6902). + PatchTypeJSONPatch = "jsonpatch" +) + +// PatchEntry is a set of patch operations targeting a specific generated file. +type PatchEntry struct { + // Type selects the patch driver. Defaults to "jsonpatch" if empty. + Type string `json:"type,omitempty" yaml:"type,omitempty" enum:",jsonpatch" description:"Patch driver type. Defaults to jsonpatch." default:"jsonpatch" example:"jsonpatch"` + + // Target is the output file to patch, relative to the pours directory. + Target string `json:"target" yaml:"target" required:"true" minLength:"1" description:"Target output file to patch" examples:"[\"compose.yaml\",\"signoz/deployment.yaml\",\"values.yaml\",\"telemetrystore/telemtrystore-clickhouse-0-*.yaml\"]"` + + // Operations is a list of JSON Patch (RFC 6902) operations to apply. Used by the jsonpatch driver. + Operations []PatchOperation `json:"operations" yaml:"operations" required:"true" minItems:"1" description:"JSON Patch (RFC 6902) operations to apply. Used by the jsonpatch driver."` + + _ struct{} `additionalProperties:"false"` +} + +// PatchType returns the patch type, defaulting to PatchTypeJSONPatch if empty. +func (pe PatchEntry) PatchType() string { + if pe.Type == "" { + return PatchTypeJSONPatch + } + + return pe.Type +} + +// PatchOperation is a single JSON Patch (RFC 6902) operation. Used by the jsonpatch driver. +type PatchOperation struct { + // Op is the JSON Patch (RFC 6902) operation type: add, remove, replace, move, copy, test. + Op string `json:"op" yaml:"op" required:"true" enum:"add,remove,replace,move,copy,test" description:"JSON Patch (RFC 6902) operation type"` + + // Path is a JSON Pointer (RFC 6902) to the target location. + Path string `json:"path" yaml:"path" required:"true" pattern:"^/" description:"JSON Pointer (RFC 6901) to the target location" example:"/services/clickhouse/mem_limit"` + + // Value is the value for add, replace, or test operations. + Value any `json:"value,omitempty" yaml:"value,omitempty" description:"Value for add, replace, or test operations"` + + // From is a JSON Pointer for the source location in move and copy operations. + From string `json:"from,omitempty" yaml:"from,omitempty" pattern:"^/" description:"Source JSON Pointer for move and copy operations" example:"/services/clickhouse/old_field"` + + _ struct{} `additionalProperties:"false"` +} diff --git a/api/v1alpha1/pointer.go b/api/v1alpha1/pointer.go new file mode 100644 index 0000000..4cf169b --- /dev/null +++ b/api/v1alpha1/pointer.go @@ -0,0 +1,9 @@ +package v1alpha1 + +func BoolPtr(v bool) *bool { + return &v +} + +func IntPtr(v int) *int { + return &v +} diff --git a/api/v1alpha1/schema.go b/api/v1alpha1/schema.go new file mode 100644 index 0000000..931bba0 --- /dev/null +++ b/api/v1alpha1/schema.go @@ -0,0 +1,20 @@ +package v1alpha1 + +import ( + "encoding/json" + + "github.com/google/jsonschema-go/jsonschema" +) + +// MustResolveSchema parses and resolves a JSON Schema, panicking on failure. +func MustResolveSchema(bytes []byte) *jsonschema.Resolved { + s := new(jsonschema.Schema) + if err := json.Unmarshal(bytes, s); err != nil { + panic(err) + } + resolved, err := s.Resolve(nil) + if err != nil { + panic(err) + } + return resolved +} diff --git a/api/v1alpha1/schema_test.go b/api/v1alpha1/schema_test.go new file mode 100644 index 0000000..e9d2c83 --- /dev/null +++ b/api/v1alpha1/schema_test.go @@ -0,0 +1,23 @@ +package v1alpha1 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMustResolveSchema(t *testing.T) { + t.Parallel() + + t.Run("valid schema", func(t *testing.T) { + t.Parallel() + assert.NotNil(t, MustResolveSchema([]byte(`{"type": "object"}`))) + }) + + t.Run("invalid json panics", func(t *testing.T) { + t.Parallel() + assert.Panics(t, func() { + MustResolveSchema([]byte(`not json`)) + }) + }) +} diff --git a/cmd/foundryctl/cast.go b/cmd/foundryctl/cast.go index cd93d81..1054703 100644 --- a/cmd/foundryctl/cast.go +++ b/cmd/foundryctl/cast.go @@ -3,7 +3,6 @@ package main import ( "context" - "fmt" "log/slog" "path/filepath" @@ -17,50 +16,48 @@ func registerCastCmd(rootCmd *cobra.Command) { castCmd := &cobra.Command{ Use: "cast", Short: "Cast to the target environment.", - RunE: func(cmd *cobra.Command, args []string) error { + RunE: recoverRunE(domain.EventCast, func(cmd *cobra.Command, args []string) (domain.Properties, error) { ctx := cmd.Context() - logger := instrumentation.NewLogger(commonCfg.Debug) if !castCfg.NoGauge { - err := runGauge(ctx, logger, commonCfg.File) - if err != nil { - return err + if props, err := runGauge(ctx, rootLogger, commonCfg.File); err != nil { + return props, err } } if !castCfg.NoForge { - err := runForge(ctx, logger, commonCfg.File, poursCfg.Path) - if err != nil { - return err + if props, err := runForge(ctx, rootLogger, commonCfg.File, poursCfg.Path); err != nil { + return props, err } } - return runCast(ctx, logger, poursCfg.Path, commonCfg.File) - }, + return runCast(ctx, rootLogger, poursCfg.Path, commonCfg.File) + }), } rootCmd.AddCommand(castCmd) castCfg.RegisterFlags(castCmd) } -func runCast(ctx context.Context, logger *slog.Logger, poursPath string, configPath string) error { +func runCast(ctx context.Context, logger *slog.Logger, poursPath string, configPath string) (domain.Properties, error) { foundry, err := foundry.New(logger) if err != nil { logger.ErrorContext(ctx, "failed to create foundry, please report this issues to developers at https://github.com/hanzoai/o11y-foundry/issues", foundryerrors.LogAttr(err)) return err } - // Get absolute pours path poursPath, err = filepath.Abs(poursPath) if err != nil { - return fmt.Errorf("failed to resolve pours path: %w", err) + return domain.NewProperties(), errors.Wrapf(err, errors.TypeInternal, "failed to resolve pours path") } - lock, err := foundry.Config.GetV1Alpha1Lock(ctx, configPath) + machinery, err := foundry.Config.GetV1Alpha1Lock(ctx, configPath) if err != nil { - logger.ErrorContext(ctx, "failed to load generated casting.yaml.lock", foundryerrors.LogAttr(err)) - return err + return domain.NewProperties(), err } - return foundry.Cast(ctx, lock, poursPath) + props := machinery.TrackableProperties() + + err = foundry.Cast(ctx, machinery, poursPath) + return props, err } diff --git a/cmd/foundryctl/catalog.go b/cmd/foundryctl/catalog.go new file mode 100644 index 0000000..8a75b4f --- /dev/null +++ b/cmd/foundryctl/catalog.go @@ -0,0 +1,106 @@ +package main + +import ( + "encoding/json" + "log/slog" + "os" + "sort" + + "github.com/olekukonko/tablewriter" + "github.com/signoz/foundry/api/v1alpha1" + installationcasting "github.com/signoz/foundry/internal/casting/installation" + "github.com/signoz/foundry/internal/domain" + "github.com/spf13/cobra" +) + +func registerCatalogCmd(rootCmd *cobra.Command) { + catalogCmd := &cobra.Command{ + Use: "catalog", + Short: "Show available castings", + RunE: recoverRunE(domain.EventCatalog, func(cmd *cobra.Command, args []string) (domain.Properties, error) { + return runCatalog(rootLogger) + }), + } + + catalogCfg.RegisterFlags(catalogCmd) + rootCmd.AddCommand(catalogCmd) +} + +type castingEntry struct { + Platform string `json:"platform"` + Mode string `json:"mode"` + Flavor string `json:"flavor"` + Example string `json:"example"` +} + +func castingExample(d v1alpha1.TypeDeployment) string { + platform, mode, flavor := d.Platform.String(), d.Mode.String(), d.Flavor.String() + switch { + case platform != "" && mode != "": + return platform + "/" + mode + "/" + flavor + case platform != "": + return platform + "/" + flavor + default: + return mode + "/" + flavor + } +} + +// catalogGroup returns a sort key that groups entries: +// 0 = mode + flavor (self-hosted: docker, systemd, kubernetes). +// 1 = mode + flavor + platform (cloud infra: ecs). +// 2 = platform + flavor (managed platforms: coolify, render, railway). +func catalogGroup(e castingEntry) int { + switch { + case e.Platform != "" && e.Mode != "": + return 1 + case e.Platform != "": + return 2 + default: + return 0 + } +} + +func runCatalog(logger *slog.Logger) (domain.Properties, error) { + registry := installationcasting.NewRegistry(logger) + + var entries []castingEntry + for d := range registry.CastingItems() { + entries = append(entries, castingEntry{ + Platform: d.Platform.String(), + Mode: d.Mode.String(), + Flavor: d.Flavor.String(), + Example: castingExample(d), + }) + } + + sort.Slice(entries, func(i, j int) bool { + gi, gj := catalogGroup(entries[i]), catalogGroup(entries[j]) + if gi != gj { + return gi < gj + } + return entries[i].Example < entries[j].Example + }) + + props := domain.NewProperties() + + if commonCfg.Format == "text" { + table := tablewriter.NewWriter(os.Stdout) + table.Header("Mode", "Flavor", "Platform", "Example") + for _, e := range entries { + _ = table.Append(e.Mode, e.Flavor, e.Platform, e.Example) + } + + return props, table.Render() + } + + data, err := json.MarshalIndent(map[string]any{"Castings": entries}, "", " ") + if err != nil { + return props, err + } + if catalogCfg.OutPath != "" { + err = os.WriteFile(catalogCfg.OutPath, data, 0644) + return props, err + } + _, err = os.Stdout.Write(data) + return props, err +} diff --git a/cmd/foundryctl/config.go b/cmd/foundryctl/config.go index 77ed614..e39d0a8 100644 --- a/cmd/foundryctl/config.go +++ b/cmd/foundryctl/config.go @@ -7,21 +7,27 @@ var ( commonCfg commonConfig // Stores pours configuration. - poursCfg poursConfig // Stores cast configuration. castCfg castConfig + + // Stores catalog configuration. + catalogCfg catalogConfig ) type commonConfig struct { - File string - Debug bool + File string + Debug bool + Format string + NoLedger bool } func (c *commonConfig) RegisterFlags(cmd *cobra.Command) { cmd.PersistentFlags().StringVarP(&c.File, "file", "f", "casting.yaml", "Path to the casting configuration file.") cmd.PersistentFlags().BoolVarP(&c.Debug, "debug", "d", false, "Enable debug mode.") + cmd.PersistentFlags().StringVar(&c.Format, "format", "json", "Output format for results and errors (json|text).") + cmd.PersistentFlags().BoolVar(&c.NoLedger, "no-ledger", false, "Disable anonymous usage ledger.") } type poursConfig struct { @@ -41,3 +47,11 @@ func (c *castConfig) RegisterFlags(cmd *cobra.Command) { cmd.PersistentFlags().BoolVar(&c.NoGauge, "no-gauge", false, "Do not run gauge before forge and cast.") cmd.PersistentFlags().BoolVar(&c.NoForge, "no-forge", false, "Do not run forge before cast.") } + +type catalogConfig struct { + OutPath string +} + +func (c *catalogConfig) RegisterFlags(cmd *cobra.Command) { + cmd.Flags().StringVarP(&c.OutPath, "output", "o", "", "Path to write castings.json") +} diff --git a/cmd/foundryctl/forge.go b/cmd/foundryctl/forge.go index 7ee637d..73d32ec 100644 --- a/cmd/foundryctl/forge.go +++ b/cmd/foundryctl/forge.go @@ -18,38 +18,33 @@ func registerForgeCmd(rootCmd *cobra.Command) { Use: "forge", Short: "Forge Configuration and Deployment Files", Long: "Generate deployment configuration files from casting.yaml", - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - logger := instrumentation.NewLogger(commonCfg.Debug) - - return runForge(ctx, logger, commonCfg.File, poursCfg.Path) - }, + RunE: recoverRunE(domain.EventForge, func(cmd *cobra.Command, args []string) (domain.Properties, error) { + return runForge(cmd.Context(), rootLogger, commonCfg.File, poursCfg.Path) + }), } rootCmd.AddCommand(forgeCmd) } -func runForge(ctx context.Context, logger *slog.Logger, path string, poursPath string) error { +func runForge(ctx context.Context, logger *slog.Logger, path string, poursPath string) (domain.Properties, error) { foundry, err := foundry.New(logger) if err != nil { logger.ErrorContext(ctx, "failed to create foundry, please report this issues to developers at https://github.com/hanzoai/o11y-foundry/issues", foundryerrors.LogAttr(err)) return err } - config, err := foundry.Config.GetV1Alpha1(ctx, path) + machinery, err := foundry.Config.GetV1Alpha1(ctx, path) if err != nil { - return err + return domain.NewProperties(), err } + props := machinery.TrackableProperties() + poursAbsPath, err := filepath.Abs(poursPath) if err != nil { - return err + return props, err } - err = foundry.Forge(ctx, config, path, &writer.Options{Output: &os.File{}, TargetDirectory: poursAbsPath}) - if err != nil { - return err - } - - return nil + err = foundry.Forge(ctx, machinery, path, &writer.Options{Output: &os.File{}, TargetDirectory: poursAbsPath}) + return props, err } diff --git a/cmd/foundryctl/gauge.go b/cmd/foundryctl/gauge.go index e08f4d3..f846449 100644 --- a/cmd/foundryctl/gauge.go +++ b/cmd/foundryctl/gauge.go @@ -14,18 +14,15 @@ func registerGaugeCmd(rootCmd *cobra.Command) { gaugeCmd := &cobra.Command{ Use: "gauge", Short: "Gauge whether required tools are available.", - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - logger := instrumentation.NewLogger(commonCfg.Debug) - - return runGauge(ctx, logger, commonCfg.File) - }, + RunE: recoverRunE(domain.EventGauge, func(cmd *cobra.Command, args []string) (domain.Properties, error) { + return runGauge(cmd.Context(), rootLogger, commonCfg.File) + }), } rootCmd.AddCommand(gaugeCmd) } -func runGauge(ctx context.Context, logger *slog.Logger, path string) error { +func runGauge(ctx context.Context, logger *slog.Logger, path string) (domain.Properties, error) { foundry, err := foundry.New(logger) if err != nil { logger.ErrorContext(ctx, "failed to create foundry, please report this issues to developers at https://github.com/hanzoai/o11y-foundry/issues", foundryerrors.LogAttr(err)) @@ -34,15 +31,11 @@ func runGauge(ctx context.Context, logger *slog.Logger, path string) error { casting, err := foundry.Config.GetV1Alpha1(ctx, path) if err != nil { - logger.ErrorContext(ctx, err.Error()) - return err + return domain.NewProperties(), err } + props := casting.TrackableProperties() + err = foundry.Gauge(ctx, casting) - if err != nil { - logger.ErrorContext(ctx, err.Error()) - return err - } - - return nil + return props, err } diff --git a/cmd/foundryctl/gen.go b/cmd/foundryctl/gen.go index c8c3286..dea3bd2 100644 --- a/cmd/foundryctl/gen.go +++ b/cmd/foundryctl/gen.go @@ -2,10 +2,12 @@ package main import ( "context" - "log" + "encoding/json" "log/slog" "os" "path/filepath" + "reflect" + "strings" "github.com/hanzoai/o11y-foundry/api/v1alpha1" foundryerrors "github.com/hanzoai/o11y-foundry/internal/errors" @@ -16,6 +18,18 @@ import ( "github.com/swaggest/jsonschema-go" ) +const moduleAPIPrefix = "github.com/signoz/foundry/api/v1alpha1/" + +type schemaTarget struct { + kind v1alpha1.Kind + val any +} + +var schemaTargets = []schemaTarget{ + {v1alpha1.KindInstallation, installation.Casting{}}, + {v1alpha1.KindCollectionAgent, collectionagent.Casting{}}, +} + func registerGenCmd(rootCmd *cobra.Command) { genCmd := &cobra.Command{ Use: "gen", @@ -48,10 +62,7 @@ func registerGenSchemas(rootCmd *cobra.Command) { Use: "schemas", Short: "Generate schema files.", RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - logger := instrumentation.NewLogger(commonCfg.Debug) - - return runGenSchemas(ctx, logger) + return runGenSchemas(cmd.Context()) }, } @@ -65,25 +76,22 @@ func runGenExamples(ctx context.Context, logger *slog.Logger) error { return err } - for deployment := range foundry.Registry.CastingItems() { + for deployment := range registry.CastingItems() { logger.InfoContext(ctx, "generating example files for deployment", slog.Any("deployment", deployment)) - config := v1alpha1.ExampleCasting() + config := installation.Example() config.Spec.Deployment = deployment - rootPath := filepath.Join("docs", "examples/", deployment.Platform, deployment.Mode, deployment.Flavor) - err = os.MkdirAll(rootPath, 0755) - if err != nil { + rootPath := filepath.Join("docs", "examples/", deployment.Platform.String(), deployment.Mode.String(), deployment.Flavor.String()) + if err := os.MkdirAll(rootPath, 0755); err != nil { return err } - err = os.WriteFile(filepath.Join(rootPath, "casting.yaml"), types.MustMarshalYAML(config), 0644) - if err != nil { + if err := os.WriteFile(filepath.Join(rootPath, "casting.yaml"), domain.MustMarshalYAML(config), 0644); err != nil { return err } - err = runForge(ctx, logger, filepath.Join(rootPath, "casting.yaml"), filepath.Join(rootPath, "pours")) - if err != nil { + if _, err := runForge(ctx, logger, filepath.Join(rootPath, "casting.yaml"), filepath.Join(rootPath, "pours")); err != nil { logger.ErrorContext(ctx, "failed to forge casting", slog.Any("deployment", deployment), foundryerrors.LogAttr(err)) continue } @@ -92,18 +100,52 @@ func runGenExamples(ctx context.Context, logger *slog.Logger) error { return nil } -func runGenSchemas(context.Context, *slog.Logger) error { - reflector := jsonschema.Reflector{} +func runGenSchemas(_ context.Context) error { + var oneOf []jsonschema.SchemaOrBool + kindType := reflect.TypeFor[v1alpha1.Kind]() - schema, err := reflector.Reflect(v1alpha1.Casting{}) - if err != nil { - log.Fatal(err) + for _, t := range schemaTargets { + target := t + reflector := jsonschema.Reflector{} + // v1alpha1.Kind's Enum() returns all Kinds (the type permits any). + // For this per-Kind schema, the kind field is always this Casting's + // Kind, so we narrow the enum at reflection. + reflector.DefaultOptions = append(reflector.DefaultOptions, + jsonschema.InterceptSchema(func(params jsonschema.InterceptSchemaParams) (bool, error) { + if !params.Processed || params.Value.Type() != kindType { + return false, nil + } + params.Schema.Enum = []any{target.kind.String()} + return false, nil + }), + ) + + schema, err := reflector.Reflect(target.val) + if err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "reflect %T", target.val) + } + + contents, err := json.MarshalIndent(schema, "", " ") + if err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "marshal %T", target.val) + } + + kindDir := strings.TrimPrefix(reflect.TypeOf(target.val).PkgPath(), moduleAPIPrefix) + if err := os.WriteFile(filepath.Join("api", "v1alpha1", kindDir, "casting.schema.json"), contents, 0644); err != nil { + return err + } + ref := (&jsonschema.Schema{}).WithRef(filepath.Join(kindDir, "casting.schema.json")) + oneOf = append(oneOf, ref.ToSchemaOrBool()) } - err = os.WriteFile(filepath.Join("docs", "schemas", "v1alpha1.yaml"), types.MustMarshalYAML(schema), 0644) + root := (&jsonschema.Schema{}). + WithSchema("http://json-schema.org/draft-07/schema#"). + WithTitle("Foundry Casting"). + WithOneOf(oneOf...) + + rootContents, err := json.MarshalIndent(root, "", " ") if err != nil { return err } - - return nil + return os.WriteFile(filepath.Join("api", "v1alpha1", "casting.schema.json"), rootContents, 0644) } diff --git a/cmd/foundryctl/main.go b/cmd/foundryctl/main.go index 35eb7d7..c10dfef 100644 --- a/cmd/foundryctl/main.go +++ b/cmd/foundryctl/main.go @@ -1,7 +1,6 @@ package main import ( - "context" "os" foundryerrors "github.com/hanzoai/o11y-foundry/internal/errors" @@ -11,12 +10,12 @@ import ( func main() { rootCmd := &cobra.Command{ - Use: "foundryctl", - SilenceUsage: true, - SilenceErrors: true, + Use: "foundryctl", + SilenceUsage: true, CompletionOptions: cobra.CompletionOptions{ DisableDefaultCmd: true, }, + PersistentPreRunE: newRoot, } // Register configuration. @@ -28,11 +27,12 @@ func main() { registerForgeCmd(rootCmd) registerCastCmd(rootCmd) registerGenCmd(rootCmd) + registerCatalogCmd(rootCmd) + registerVersionCmd(rootCmd) - logger := instrumentation.NewLogger(false) + defer closeRoot() if err := rootCmd.Execute(); err != nil { - logger.ErrorContext(context.Background(), "failed to run foundryctl", foundryerrors.LogAttr(err)) - os.Exit(1) + os.Exit(foundryerrors.ExitCode(err)) } } diff --git a/cmd/foundryctl/root.go b/cmd/foundryctl/root.go new file mode 100644 index 0000000..a162ff0 --- /dev/null +++ b/cmd/foundryctl/root.go @@ -0,0 +1,82 @@ +package main + +import ( + "log/slog" + "os" + "runtime" + + "github.com/signoz/foundry/internal/domain" + foundryerrors "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/instrumentation" + "github.com/signoz/foundry/internal/ledger" + "github.com/signoz/foundry/internal/ledger/noopledger" + "github.com/signoz/foundry/internal/ledger/segmentledger" + "github.com/signoz/foundry/internal/writer" + "github.com/spf13/cobra" +) + +var ( + rootLogger *slog.Logger + rootTracker ledger.Ledger +) + +// newRoot is wired as rootCmd.PersistentPreRunE so it fires after persistent +// flags are parsed and before any command's RunE runs. +func newRoot(_ *cobra.Command, _ []string) error { + rootLogger = instrumentation.NewLogger(commonCfg.Debug) + + config := ledger.NewConfig() + if commonCfg.NoLedger { + config.Enabled = false + } + + switch config.Provider() { + case "segment": + rootTracker = segmentledger.New(config) + default: + rootTracker = noopledger.New() + } + + return nil +} + +func closeRoot() { + if rootTracker != nil { + _ = rootTracker.Close() + } +} + +func recoverRunE( + event domain.Event, + runE func(cmd *cobra.Command, args []string) (domain.Properties, error), +) func(cmd *cobra.Command, args []string) error { + return func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + props := domain.NewProperties() + + defer func() { + if r := recover(); r != nil { + buf := make([]byte, 4096) + n := runtime.Stack(buf, false) + + errp := foundryerrors.Newf(foundryerrors.TypeFatal, "%v", r).WithStacktrace(string(buf[:n])) + err = errp + } + + if err != nil { + rootLogger.ErrorContext(ctx, event.String()+" failed", foundryerrors.LogAttr(err)) + rootTracker.Track(ctx, event.Failed(), props.WithError(err)) + if commonCfg.Format == "json" { + _ = writer.WriteOutput(os.Stdout, foundryerrors.EnvelopeOf(err)) + cmd.SilenceErrors = true + } + return + } + + rootTracker.Track(ctx, event.Succeeded(), props.WithSuccess()) + }() + + props, err = runE(cmd, args) + return err + } +} diff --git a/cmd/foundryctl/version.go b/cmd/foundryctl/version.go new file mode 100644 index 0000000..2bbb647 --- /dev/null +++ b/cmd/foundryctl/version.go @@ -0,0 +1,36 @@ +package main + +import ( + "fmt" + "os" + + "github.com/signoz/foundry/internal/version" + "github.com/spf13/cobra" +) + +var versionCfg versionConfig + +type versionConfig struct { + Short bool +} + +func (c *versionConfig) RegisterFlags(cmd *cobra.Command) { + cmd.Flags().BoolVar(&c.Short, "short", false, "Print version in a single line") +} + +func registerVersionCmd(rootCmd *cobra.Command) { + versionCmd := &cobra.Command{ + Use: "version", + Short: "Print the version information", + Run: func(cmd *cobra.Command, args []string) { + if versionCfg.Short { + _, _ = fmt.Fprintln(os.Stdout, version.Info.Short()) + return + } + version.Info.PrettyPrint() + }, + } + + versionCfg.RegisterFlags(versionCmd) + rootCmd.AddCommand(versionCmd) +} diff --git a/docs/casting.md b/docs/casting.md deleted file mode 100644 index a67b445..0000000 --- a/docs/casting.md +++ /dev/null @@ -1,173 +0,0 @@ -## Casting - -A _casting_ is one YAML file that describes a full Hanzo O11y deployment. Foundry fills in defaults; you override what you need. - -### How to write `casting.yaml` - -You’ll build the file in this order: - -1. **Name your deployment**: `apiVersion` and `metadata` (name, optional annotations). -2. **Where it runs**: Deployment target: Docker, systemd, or Render. -3. **What runs**: Moldings (Hanzo O11y, ingester, ClickHouse, metastore). Add blocks when you want to change defaults. -4. **How it's configured**: Per-molding `spec`: images, env, scaling, config files. -5. **Run it**: Point Foundry at the file and generate artifacts. - - -#### 1. Name your deployment - -Top of the file: `apiVersion` and `metadata`. - -```yaml -apiVersion: v1alpha1 -metadata: - name: o11y-prod # deployment ID: used as prefix for service names - annotations: {} # optional; required for systemd (step 2) -``` - -`name` is required: use something that identifies this deployment (`o11y-prod`, `o11y-dev`, whatever). `annotations` is optional unless you're on systemd/binary; then you'll put binary paths there (see step 2). - -> [!TIP] -> Short, environment-specific names work best; they end up in generated service names. - -#### 2. Where it runs (deployment target) - -`spec.deployment` tells Foundry how you're deploying: Docker Compose, systemd, or Render. It uses this to pick the right mold and spit out the right artifacts. - -```yaml -spec: - deployment: - mode: docker # or systemd - flavor: compose # or binary | blueprint - platform: # optional; "render" for Render -``` - -Pick one row: - -| Where you're deploying | `mode` | `flavor` | `platform` | -| ---------------------- | ---------- | ----------- | --------- | -| **Docker Compose** | `docker` | `compose` | (none) | -| **Linux (systemd)** | `systemd` | `binary` | (none) | -| **[Render](https://render.com)** | (none) | `blueprint` | `render` | - -> [!NOTE] -> systemd (`mode` + `flavor: binary`):** Foundry needs the paths to your binaries. Put them in `metadata.annotations`: -> -> | Annotation | What it's for | -> | ---------- | ------------- | -> | `foundry.o11y.hanzo.ai/o11y-binary-path` | Hanzo O11y binary (for example, `/opt/o11y/bin/o11y`) | -> | `foundry.o11y.hanzo.ai/ingester-binary-path` | OTel Collector / ingester (for example, `/opt/ingester/bin/o11y-otel-collector`) | -> | `foundry.o11y.hanzo.ai/metastore-postgres-binary-path` | PostgreSQL binary when using Postgres metastore (for example, `/usr/bin/postgres`) | -> -> Example: -> -> ```yaml -> metadata: -> name: o11y -> annotations: -> foundry.o11y.hanzo.ai/o11y-binary-path: /opt/o11y/bin/o11y -> foundry.o11y.hanzo.ai/ingester-binary-path: /opt/ingester/bin/o11y-otel-collector -> foundry.o11y.hanzo.ai/metastore-postgres-binary-path: /usr/bin/postgres -> ``` - -#### 3. What runs (moldings) - -_Moldings_ are the pieces (Hanzo O11y, ingester, ClickHouse, etc.). Foundry has defaults for all of them; add a block under `spec` when you want to change one. - -| Molding key in `spec` | Component | -| --------------------- | --------- | -| `o11y` | Hanzo O11y | -| `ingester` | OTel Collector (ingestion & processing) | -| `telemetrystore` | ClickHouse (logs, traces, metrics) | -| `telemetrykeeper` | ClickHouse Keeper (coordination) | -| `metastore` | Metadata store (PostgreSQL or SQLite) | - -Angle brackets are placeholders: swap `<deployment-name>` for your ID, and pick valid `mode` / `flavor` / `platform` from the table above. - -```yaml -apiVersion: v1alpha1 -metadata: - name: <deployment-name> - annotations: {} # optional; required for systemd with binary paths -spec: - deployment: - mode: <docker|systemd> - flavor: <compose|binary|blueprint> - platform: <render> # optional - # Override only what you need: - o11y: - spec: { ... } - ingester: - spec: { ... } - telemetrystore: - spec: { ... } - telemetrykeeper: - spec: { ... } - metastore: - kind: postgres # or sqlite - spec: { ... } -``` - -#### 4. How it’s configured (molding spec) - -Override a molding by giving it a `spec` block. Whatever you set gets merged with Foundry's defaults. - -**Fields you'll see:** - -| Field | Meaning | -| ------------------- | ------- | -| `enabled` | Turn the component on/off (default: `true`) | -| `image` | Container image (Docker mode) | -| `version` | Version label (for example, for systemd or tagging) | -| `cluster.replicas` | Number of replicas | -| `cluster.shards` | Shards (TelemetryStore only) | -| `env` | Environment variables (key/value map) | -| `config.data` | Config files: **filename → file contents** | - -#### 5. Run it - -When the file's done: - -1. Run: - - ```shell - foundry cast -f casting.yaml - ``` - -2. Foundry merges your overrides with defaults and writes out the artifacts (Compose files, systemd units, or Render blueprint, depending on what you picked). - -That's it. The casting file is the source of truth; Foundry does the rest. - -## Examples - -**Minimal: Docker Compose, all defaults:** - -```yaml -apiVersion: v1alpha1 -metadata: - name: o11y -spec: - deployment: - mode: docker - flavor: compose -``` - -**Same, with a few overrides (images, scaling, env):** - -```yaml -apiVersion: v1alpha1 -metadata: - name: o11y -spec: - deployment: - mode: docker - flavor: compose - o11y: - spec: - image: ghcr.io/hanzoai/o11y:v0.110.0 - telemetrystore: - spec: - image: ghcr.io/hanzoai/datastore:25.5.6 - cluster: - replicas: 1 - shards: 1 -``` diff --git a/docs/concepts/annotations.md b/docs/concepts/annotations.md new file mode 100644 index 0000000..bdda900 --- /dev/null +++ b/docs/concepts/annotations.md @@ -0,0 +1,58 @@ +# Annotations + +Annotations are key-value pairs in `metadata.annotations` that provide deployment-specific configuration to Foundry. They are inputs to the generation process - Foundry reads them during forge and uses them to populate templates and configure behavior. + +## How annotations work + +Annotations live in the casting file's metadata block: + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz + annotations: + foundry.signoz.io/some-key: some-value +spec: + deployment: + mode: systemd + flavor: binary +``` + +During forge, Foundry reads annotation values and injects them into the generated output. For example, systemd annotations set binary paths in the generated `.service` files. ECS annotations populate Terraform variables in the generated `terraform.tfvars.json`. + +## Annotations vs patches + +Annotations and patches serve different roles in the generation pipeline: + +| | Annotations | Patches | +|---|---|---| +| **Where** | `metadata.annotations` | `spec.patches` | +| **When** | Before generation - Foundry reads them as input | After generation - Foundry applies them to output | +| **What** | Deployment-specific parameters (paths, IDs, ARNs) that Foundry needs to generate files correctly | Modifications to already-generated files (resource limits, storage classes, scheduling) | +| **Validated** | Yes - Foundry reads and uses them during template execution | No - Foundry passes them through as-is | + +In short: annotations tell Foundry *how to generate*, patches tell Foundry *what to change after generating*. + +### When to use which + +Use **annotations** when Foundry needs the value during generation. These are typically infrastructure identifiers, file paths, or settings that determine the shape of the generated output. You cannot achieve the same result with patches because the value must exist before files are generated. + +Use **patches** when you want to modify a generated file after the fact. These are typically platform-specific tuning (resource limits, scheduling, service types) that Foundry doesn't need to understand. + +## Which castings use annotations + +Not all deployment modes require annotations. Most modes (Docker Compose, Docker Swarm, Kubernetes Kustomize, Render, Coolify, Railway) work with just `metadata.name` and `spec.deployment`. + +| Casting | Required | Annotations | +|---|---|---| +| [Systemd (binary)](../examples/systemd/binary/) | Yes | Binary paths for SigNoz, ingester, and PostgreSQL | +| [ECS EC2 (Terraform)](../examples/ecs/ec2/terraform/) | Yes | AWS region, cluster ID, subnets, security groups, IAM roles, S3 bucket | +| [Kubernetes (Helm)](../examples/kubernetes/helm/) | No | Optional chart repo and chart name overrides | + +For the complete list of annotation keys and their descriptions, see [Casting File Reference](../reference/casting-file.md#annotations). + +## Next steps + +- [Patches](patches.md) - post-generation modifications +- [Casting](casting.md) - the full casting file structure +- [Casting file reference](../reference/casting-file.md#annotations) - complete annotation reference diff --git a/docs/concepts/casting.md b/docs/concepts/casting.md new file mode 100644 index 0000000..7a736fa --- /dev/null +++ b/docs/concepts/casting.md @@ -0,0 +1,144 @@ +# Casting + +A casting is one YAML file that describes a complete SigNoz deployment. Foundry reads it, merges your overrides with built-in defaults, and generates everything needed to run the stack. + +The casting file is the single source of truth for your deployment. You define what you want; Foundry handles the rest. + +## Structure + +A casting has five parts: + +1. **Metadata** - name your deployment and set annotations +2. **Deployment target** - where it runs (Docker, Kubernetes, systemd, cloud platform) +3. **Moldings** - which components to include and how to configure them +4. **Annotations** - deployment-specific parameters Foundry reads during generation +5. **Patches** - platform-level overrides on the generated output + +```yaml +apiVersion: v1alpha1 +metadata: + name: <deployment-name> +spec: + deployment: + mode: <mode> + flavor: <flavor> + platform: <platform> # only for cloud platforms + signoz: + spec: { ... } + ingester: + spec: { ... } + telemetrystore: + spec: { ... } + telemetrykeeper: + spec: { ... } + metastore: + kind: postgres # or sqlite + spec: { ... } + patches: [ ... ] +``` + +Replace `<deployment-name>` with an identifier for this deployment (for example, `signoz-prod`). This name is used as a prefix in generated service names. + +## Metadata + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz-prod + annotations: {} # optional; required for some deployment modes +``` + +`name` is required. Short, environment-specific names work best since they end up in generated service names. + +`annotations` is optional for most deployment modes. Some modes (systemd, ECS, Helm) use annotations to pass deployment-specific parameters. See [Annotations](annotations.md) for when and how to use them. + +## Deployment target + +`spec.deployment` tells Foundry how you're deploying. It picks the right generator and produces the right artifacts. + +```yaml +spec: + deployment: + mode: docker + flavor: compose +``` + +Each row below is a valid combination. Mixing values across rows is not supported. + +| Target | `mode` | `flavor` | `platform` | +|---|---|---|---| +| Docker Compose | `docker` | `compose` | - | +| Docker Swarm | `docker` | `swarm` | - | +| Systemd (binary) | `systemd` | `binary` | - | +| Kubernetes (Kustomize) | `kubernetes` | `kustomize` | - | +| Kubernetes (Helm) | `kubernetes` | `helm` | - | +| Render | - | `blueprint` | `render` | +| Coolify | - | `stack` | `coolify` | +| Railway | - | `template` | `railway` | +| AWS ECS (EC2) | `ec2` | `terraform` | `ecs` | + +> [!TIP] +> Run `foundryctl gen examples` to generate a working `casting.yaml` for every supported deployment mode. + +## Moldings + +Moldings are the individual components of a SigNoz deployment. Foundry has defaults for all of them. Add a block under `spec` only when you want to change something. + +See [Moldings](moldings.md) for details on each component and how to configure them. + +## Annotations + +Annotations provide deployment-specific parameters that Foundry reads during generation. They are inputs to the pipeline, not modifications to the output. + +Most deployment modes don't require annotations. Systemd and ECS do. Helm supports optional chart overrides. + +See [Annotations](annotations.md) for the full guide, including how annotations differ from patches. + +## Patches + +Patches let you customize any generated output file without Foundry needing to model every platform-specific field. They use JSON Patch (RFC 6902) operations. + +See [Patches](patches.md) for the full guide. + +## Minimal example + +Docker Compose with all defaults: + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + mode: docker + flavor: compose +``` + +With overrides for images and scaling: + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + mode: docker + flavor: compose + signoz: + spec: + image: signoz/signoz:v0.110.0 + telemetrystore: + spec: + image: clickhouse/clickhouse-server:25.5.6 + cluster: + replicas: 1 + shards: 1 +``` + +## Next steps + +- [Moldings](moldings.md) - configure individual components +- [Annotations](annotations.md) - deployment-specific parameters +- [Patches](patches.md) - platform-level overrides +- [Casting file reference](../reference/casting-file.md) - complete field-by-field reference +- [Examples](../examples/) - working examples for every deployment mode diff --git a/docs/concepts/moldings.md b/docs/concepts/moldings.md new file mode 100644 index 0000000..7f7a5a3 --- /dev/null +++ b/docs/concepts/moldings.md @@ -0,0 +1,107 @@ +# Moldings + +Moldings are the individual components that make up a SigNoz deployment. Each molding maps to a service (ClickHouse, PostgreSQL, OTel Collector, etc.) and generates the configuration and deployment files for that service. + +Foundry ships with sensible defaults for every molding. Override only what you need by adding a block under `spec` in your casting. + +## Components + +| Key in `spec` | Component | Role | +|---|---|---| +| `telemetrykeeper` | ClickHouse Keeper | Cluster coordination for ClickHouse | +| `telemetrystore` | ClickHouse | Stores logs, traces, and metrics | +| `metastore` | PostgreSQL or SQLite | Stores metadata (dashboards, alerts, users) | +| `signoz` | SigNoz | UI and API server | +| `ingester` | SigNoz OTel Collector | Receives, processes, and writes telemetry data | + +### Processing order + +Foundry processes moldings in dependency order during forge: + +``` +TelemetryKeeper -> TelemetryStore -> MetaStore -> SigNoz -> Ingester +``` + +This ensures each component's configuration can reference its dependencies. + +## Configuring a molding + +Override a molding by giving it a `spec` block. Whatever you set gets merged with Foundry's defaults. + +```yaml +spec: + telemetrystore: + spec: + image: clickhouse/clickhouse-server:25.5.6 + cluster: + replicas: 1 + shards: 1 +``` + +### Spec fields + +| Field | Description | +|---|---| +| `enabled` | Turn the component on or off (default: `true`) | +| `image` | Container image (Docker and Kubernetes modes) | +| `version` | Version label (systemd mode, tagging) | +| `cluster.replicas` | Number of replicas | +| `cluster.shards` | Number of shards (TelemetryStore only) | +| `env` | Environment variables as a key-value map | +| `config.data` | Config file overrides: filename to file contents | + +### Disabling a molding + +Set `enabled: false` to exclude a component from the deployment: + +```yaml +spec: + telemetrykeeper: + spec: + enabled: false +``` + +### Custom config files + +Use `config.data` to override the contents of a component's config files. The key is the filename; the value is the file contents. + +```yaml +spec: + ingester: + spec: + config: + data: + ingester.yaml: | + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 +``` + +> [!NOTE] +> `config.data` overrides application-level config files that Foundry understands and manages. For platform-level files (compose files, service units, Kubernetes manifests), use [patches](patches.md) instead. + +### MetaStore kind + +The metastore supports two backends. Set the `kind` field to choose: + +```yaml +spec: + metastore: + kind: postgres # default + spec: + image: postgres:16 +``` + +| Kind | Backend | Notes | +|---|---|---| +| `postgres` | PostgreSQL | Default. Recommended for production. | +| `sqlite` | SQLite | Embedded, no external dependency. Single-node only. | + +## Next steps + +- [Casting](casting.md) - the full casting file structure +- [Annotations](annotations.md) - deployment-specific parameters +- [Patches](patches.md) - platform-level overrides on generated files +- [Casting file reference](../reference/casting-file.md) - complete field reference diff --git a/docs/concepts/patches.md b/docs/concepts/patches.md new file mode 100644 index 0000000..dea50f1 --- /dev/null +++ b/docs/concepts/patches.md @@ -0,0 +1,120 @@ +# Patches + +Patches let you customize any generated output file using JSON Patch (RFC 6902) operations. They are applied during forge, after Foundry generates materials and before writing to `pours/`. + +## Two-tier model + +Foundry separates configuration into two layers: + +- **`spec.<Molding>`** is the application domain. Foundry understands, validates, and enriches it. +- **`spec.patches`** is the platform domain. Foundry applies it as-is, no validation. + +This gives you full coverage over the generated output without Foundry needing to model every platform-specific field. Use `spec.<Molding>` for component configuration (images, replicas, env, config files). Use `patches` for everything else (resource limits, storage classes, service types, scheduling constraints). + +## Patch entry fields + +| Field | Required | Description | +|---|---|---| +| `type` | No | Patch driver. Defaults to `jsonpatch`. | +| `target` | Yes | Output file to patch. Supports exact path, basename, or glob. | +| `operations` | Yes | List of JSON Patch (RFC 6902) operations. | + +## JSON Patch operations + +| Operation | Description | Required fields | +|---|---|---| +| `add` | Add a value at path. Append to array with `/-`. | `op`, `path`, `value` | +| `remove` | Remove the value at path. | `op`, `path` | +| `replace` | Replace the value at path. Path must exist. | `op`, `path`, `value` | +| `move` | Move a value from one path to another. | `op`, `from`, `path` | +| `copy` | Copy a value from one path to another. | `op`, `from`, `path` | +| `test` | Assert a value equals the given value. Fails if not. | `op`, `path`, `value` | + +## Target matching + +- **Exact path:** `target: "deployment/compose.yaml"` +- **Glob:** `target: "deployment/telemetrystore-*.yaml"` matches multiple files + +> [!TIP] +> Run `foundryctl forge` first without patches to see the generated file names and structure, then write patches against them. + +## Examples + +### Docker Compose + +Set a memory limit on ClickHouse and add a custom environment variable to SigNoz: + +```yaml +spec: + patches: + - target: "compose.yaml" + operations: + - op: replace + path: /services/clickhouse/mem_limit + value: "4G" + - op: add + path: /services/signoz/environment/- + value: "CUSTOM_VAR=value" +``` + +### Systemd + +Change the restart policy and add a memory limit on a service unit: + +```yaml +spec: + patches: + - target: "signoz-ingester.service" + operations: + - op: replace + path: /Service/Restart + value: always + - op: add + path: /Service/MemoryMax + value: "4G" +``` + +### Kubernetes (Kustomize) + +Set resource limits on ClickHouse: + +```yaml +spec: + patches: + - target: "deployment/telemetrystore/clickhouse/clickhouseinstallation.yaml" + operations: + - op: replace + path: /spec/templates/podTemplates/0/spec/containers/0/resources + value: + requests: + cpu: "2" + memory: "4Gi" + limits: + cpu: "4" + memory: "8Gi" +``` + +For complete Kubernetes examples with storage classes, tolerations, nodeSelector, and LoadBalancer configuration, see: +- [Kustomize with patches](../examples/kubernetes/kustomize-patches/) +- [Helm with patches](../examples/kubernetes/helm-patches/) + +## When to use patches vs config.data + +| Use case | Use | +|---|---| +| Application config files (OTel Collector config, ClickHouse config) | `config.data` in the molding spec | +| Platform files (compose files, service units, Kubernetes manifests, Helm values) | `spec.patches` | + +> [!NOTE] +> Config files like `otel-collector-config.yaml` or `clickhouse-config.yaml` don't need patches. Use `config.data` in the [molding spec](moldings.md#custom-config-files) instead. + +## Patches vs annotations + +Patches modify generated output *after* generation. [Annotations](annotations.md) provide parameters *before* generation. If Foundry needs a value to generate files correctly (binary paths, AWS resource IDs), use annotations. If you want to tweak something in the generated output (resource limits, service types), use patches. + +## Next steps + +- [Annotations](annotations.md) - deployment-specific parameters (pre-generation) +- [Casting](casting.md) - the full casting file structure +- [Moldings](moldings.md) - component configuration +- [Casting file reference](../reference/casting-file.md) - complete field reference diff --git a/docs/examples/README.md b/docs/examples/README.md new file mode 100644 index 0000000..d44789c --- /dev/null +++ b/docs/examples/README.md @@ -0,0 +1,13 @@ +### Examples + +| Platform | Mode | Flavor | Example | +| --- | --- | --- | --- | +| - | docker | compose | [docker/compose](docker/compose/) | +| - | docker | swarm | [docker/swarm](docker/swarm/) | +| - | kubernetes | helm | [kubernetes/helm](kubernetes/helm/) | +| - | kubernetes | kustomize | [kubernetes/kustomize](kubernetes/kustomize/) | +| - | systemd | binary | [systemd/binary](systemd/binary/) | +| ecs | ec2 | terraform | [ecs/ec2/terraform](ecs/ec2/terraform/) | +| coolify | - | stack | [coolify/stack](coolify/stack/) | +| railway | - | template | [railway/template](railway/template/) | +| render | - | blueprint | [render/blueprint](render/blueprint/) | diff --git a/docs/examples/README.md.gotmpl b/docs/examples/README.md.gotmpl new file mode 100644 index 0000000..0c05203 --- /dev/null +++ b/docs/examples/README.md.gotmpl @@ -0,0 +1,7 @@ +### Examples + +| Platform | Mode | Flavor | Example | +| --- | --- | --- | --- | +{{- range .Castings }} +| {{ or .platform "-" }} | {{ or .mode "-" }} | {{ or .flavor "-" }} | [{{ .example }}]({{ .example }}/) | +{{- end }} diff --git a/docs/examples/castings.json b/docs/examples/castings.json new file mode 100644 index 0000000..ded6771 --- /dev/null +++ b/docs/examples/castings.json @@ -0,0 +1,58 @@ +{ + "Castings": [ + { + "platform": "", + "mode": "docker", + "flavor": "compose", + "example": "docker/compose" + }, + { + "platform": "", + "mode": "docker", + "flavor": "swarm", + "example": "docker/swarm" + }, + { + "platform": "", + "mode": "kubernetes", + "flavor": "helm", + "example": "kubernetes/helm" + }, + { + "platform": "", + "mode": "kubernetes", + "flavor": "kustomize", + "example": "kubernetes/kustomize" + }, + { + "platform": "", + "mode": "systemd", + "flavor": "binary", + "example": "systemd/binary" + }, + { + "platform": "ecs", + "mode": "ec2", + "flavor": "terraform", + "example": "ecs/ec2/terraform" + }, + { + "platform": "coolify", + "mode": "", + "flavor": "stack", + "example": "coolify/stack" + }, + { + "platform": "railway", + "mode": "", + "flavor": "template", + "example": "railway/template" + }, + { + "platform": "render", + "mode": "", + "flavor": "blueprint", + "example": "render/blueprint" + } + ] +} \ No newline at end of file diff --git a/docs/examples/coolify/README.md b/docs/examples/coolify/README.md new file mode 100644 index 0000000..fb88e23 --- /dev/null +++ b/docs/examples/coolify/README.md @@ -0,0 +1,5 @@ +# Coolify + +| Flavor | Description | +| --- | --- | +| [stack](stack/) | Deploy SigNoz on Coolify as a stack | diff --git a/docs/examples/coolify/stack/README.md b/docs/examples/coolify/stack/README.md new file mode 100644 index 0000000..c0ee3ed --- /dev/null +++ b/docs/examples/coolify/stack/README.md @@ -0,0 +1,50 @@ +# Coolify Stack + +| Field | Value | +| --- | --- | +| **Mode** | `-` | +| **Flavor** | `stack` | +| **Platform** | `coolify` | + +## Overview + +Generates a `coolify.yaml` stack definition for deploying SigNoz on a Coolify-managed server. Deployment is manual via the Coolify dashboard. + +> [!NOTE] +> `foundryctl cast` does not deploy to Coolify automatically. It generates the files and prints instructions for manual deployment. + +## Prerequisites + +- A [Coolify](https://coolify.io) instance (self-hosted or cloud) + +## Configuration + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + flavor: stack + platform: coolify +``` + +## Deploy + +```bash +# Generate the stack definition +foundryctl forge -f casting.yaml +``` + +After forging, deploy the generated `coolify.yaml` using the [Coolify stack feature](https://coolify.io/docs/knowledge-base/docker/compose). + +## Generated output + +```text +pours/deployment/ + coolify.yaml +``` + +## Customization + +For changes to the generated `coolify.yaml`, use [patches](../../../concepts/patches.md). diff --git a/docs/examples/coolify/stack/casting.yaml b/docs/examples/coolify/stack/casting.yaml new file mode 100644 index 0000000..07456cf --- /dev/null +++ b/docs/examples/coolify/stack/casting.yaml @@ -0,0 +1,8 @@ +apiVersion: v1alpha1 +kind: Installation +metadata: + name: signoz +spec: + deployment: + flavor: stack + platform: coolify diff --git a/docs/examples/coolify/stack/casting.yaml.lock b/docs/examples/coolify/stack/casting.yaml.lock new file mode 100644 index 0000000..31dea54 --- /dev/null +++ b/docs/examples/coolify/stack/casting.yaml.lock @@ -0,0 +1,648 @@ +apiVersion: v1alpha1 +kind: Installation +metadata: + name: signoz +spec: + deployment: + flavor: stack + platform: coolify + ingester: + spec: + cluster: + replicas: 1 + config: + data: + ingester.yaml: | + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 5s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + metadataexporter: + enabled: true + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + - metadataexporter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + - metadataexporter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + - metadataexporter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: | + server_endpoint: ws://signoz-signoz-0:4320/v1/opamp + enabled: true + env: + SIGNOZ_OTEL_COLLECTOR_TIMEOUT: 10m + image: signoz/signoz-otel-collector:latest + version: latest + status: + addresses: + otlp: + - tcp://signoz-ingester:4318 + - tcp://signoz-ingester:4317 + config: + data: + ingester.yaml: | + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 5s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + metadataexporter: + enabled: true + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + - metadataexporter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + - metadataexporter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + - metadataexporter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: | + server_endpoint: ws://signoz-signoz-0:4320/v1/opamp + env: + SIGNOZ_OTEL_COLLECTOR_TIMEOUT: 10m + metastore: + kind: postgres + spec: + cluster: + replicas: 1 + config: {} + enabled: true + env: + POSTGRES_DB: signoz + POSTGRES_PASSWORD: signoz + POSTGRES_USER: signoz + image: postgres:16 + version: "16" + status: + addresses: + dsn: + - tcp://signoz-metastore-postgres-0:5432 + config: {} + env: + POSTGRES_DB: signoz + POSTGRES_PASSWORD: signoz + POSTGRES_USER: signoz + signoz: + spec: + cluster: + replicas: 1 + config: {} + enabled: true + env: + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@signoz-metastore-postgres-0:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://signoz-telemetrystore-clickhouse-0-0:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + image: signoz/signoz:latest + version: latest + status: + addresses: + apiserver: + - tcp://signoz-signoz-0:8080 + opamp: + - ws://signoz-signoz-0:4320 + config: {} + env: + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@signoz-metastore-postgres-0:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://signoz-telemetrystore-clickhouse-0-0:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + telemetrykeeper: + kind: clickhousekeeper + spec: + cluster: + replicas: 1 + config: + data: + keeper-0.yaml: | + listen_host: 0.0.0.0 + logger: + level: information + console: true + keeper_server: + four_letter_word_white_list: "*" + coordination_settings: + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: signoz-telemetrykeeper-clickhousekeeper-0 + port: 9234 + id: 0 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 + enabled: true + image: clickhouse/clickhouse-keeper:25.5.6 + version: 25.5.6 + status: + addresses: + client: + - tcp://signoz-telemetrykeeper-clickhousekeeper-0:9181 + raft: + - tcp://signoz-telemetrykeeper-clickhousekeeper-0:9234 + config: + data: + keeper-0.yaml: | + listen_host: 0.0.0.0 + logger: + level: information + console: true + keeper_server: + four_letter_word_white_list: "*" + coordination_settings: + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: signoz-telemetrykeeper-clickhousekeeper-0 + port: 9234 + id: 0 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 + telemetrystore: + kind: clickhouse + spec: + cluster: + replicas: 0 + shards: 1 + config: + data: + config-0-0.yaml: | + path: /var/lib/clickhouse/ + tmp_path: /var/lib/clickhouse/tmp/ + user_files_path: /var/lib/clickhouse/user_files/ + format_schema_path: /var/lib/clickhouse/format_schemas/ + dictionaries_config: '*_dictionary.xml' + display_name: cluster + distributed_ddl: + path: /clickhouse/task_queue/ddl + http_port: 8123 + interserver_http_port: 9009 + listen_host: 0.0.0.0 + logger: + console: 1 + count: 10 + formatting: + type: console + level: information + size: 1000M + macros: + replica: "00" + shard: "00" + profiles: + default: + allow_simdjson: 0 + load_balancing: random + log_queries: 1 + quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 + user_directories: + users_xml: + path: users.xml + remote_servers: + cluster: + shard: + - replica: + - host: signoz-telemetrystore-clickhouse-0-0 + port: 9000 + zookeeper: + node: + - host: signoz-telemetrykeeper-clickhousekeeper-0 + port: 9181 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_thread_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_views_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + part_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + metric_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + asynchronous_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + trace_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + error_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + latency_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + session_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + text_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + zookeeper_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + tcp_port: 9000 + user_defined_executable_functions_config: '*function.yaml' + user_scripts_path: /var/lib/clickhouse/user_scripts/ + users: + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 + functions.yaml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable + enabled: true + image: clickhouse/clickhouse-server:25.5.6 + version: 25.5.6 + status: + addresses: + tcp: + - tcp://signoz-telemetrystore-clickhouse-0-0:9000 + config: + data: + config-0-0.yaml: | + path: /var/lib/clickhouse/ + tmp_path: /var/lib/clickhouse/tmp/ + user_files_path: /var/lib/clickhouse/user_files/ + format_schema_path: /var/lib/clickhouse/format_schemas/ + dictionaries_config: '*_dictionary.xml' + display_name: cluster + distributed_ddl: + path: /clickhouse/task_queue/ddl + http_port: 8123 + interserver_http_port: 9009 + listen_host: 0.0.0.0 + logger: + console: 1 + count: 10 + formatting: + type: console + level: information + size: 1000M + macros: + replica: "00" + shard: "00" + profiles: + default: + allow_simdjson: 0 + load_balancing: random + log_queries: 1 + quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 + user_directories: + users_xml: + path: users.xml + remote_servers: + cluster: + shard: + - replica: + - host: signoz-telemetrystore-clickhouse-0-0 + port: 9000 + zookeeper: + node: + - host: signoz-telemetrykeeper-clickhousekeeper-0 + port: 9181 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_thread_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_views_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + part_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + metric_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + asynchronous_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + trace_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + error_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + latency_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + session_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + text_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + zookeeper_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + tcp_port: 9000 + user_defined_executable_functions_config: '*function.yaml' + user_scripts_path: /var/lib/clickhouse/user_scripts/ + users: + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 + functions.yaml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable diff --git a/docs/examples/coolify/stack/pours/deployment/coolify.yaml b/docs/examples/coolify/stack/pours/deployment/coolify.yaml new file mode 100644 index 0000000..6ebfd68 --- /dev/null +++ b/docs/examples/coolify/stack/pours/deployment/coolify.yaml @@ -0,0 +1,484 @@ +services: + ingester: + command: + - -c + - | + /signoz-otel-collector migrate sync check && + /signoz-otel-collector --config=/etc/otel-collector-config.yaml --manager-config=/etc/opamp-config.yaml --copy-path=/var/tmp/collector-config.yaml + depends_on: + signoz-signoz-0: + condition: service_started + signoz-telemetrystore-clickhouse-0-0: + condition: service_healthy + signoz-telemetrystore-migrator: + condition: service_completed_successfully + deploy: + replicas: 1 + entrypoint: + - /bin/sh + environment: + - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux + - LOW_CARDINAL_EXCEPTION_GROUPING=false + - SIGNOZ_OTEL_COLLECTOR_TIMEOUT=10m + image: signoz/signoz-otel-collector:latest + logging: + options: + max-file: "3" + max-size: 50m + networks: + default: + aliases: + - signoz-ingester + volumes: + - content: | + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 5s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + metadataexporter: + enabled: true + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + - metadataexporter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + - metadataexporter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + - metadataexporter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + source: ./configs/ingester/ingester.yaml + target: /etc/otel-collector-config.yaml + type: bind + - content: | + server_endpoint: ws://signoz-signoz-0:4320/v1/opamp + source: ./configs/ingester/opamp.yaml + target: /etc/opamp-config.yaml + type: bind + signoz-metastore-postgres-0: + environment: + - POSTGRES_USER=signoz + - POSTGRES_PASSWORD=signoz + - POSTGRES_DB=signoz + healthcheck: + interval: 30s + retries: 3 + test: + - CMD-SHELL + - pg_isready -U signoz -d signoz + timeout: 5s + image: postgres:16 + logging: + options: + max-file: "3" + max-size: 50m + volumes: + - source: signoz-metastore-0-data + target: /var/lib/postgresql/data + type: volume + signoz-signoz-0: + container_name: signoz-signoz-0 + depends_on: + signoz-metastore-postgres-0: + condition: service_healthy + signoz-telemetrystore-clickhouse-0-0: + condition: service_healthy + signoz-telemetrystore-migrator: + condition: service_completed_successfully + environment: + - SIGNOZ_SQLSTORE_POSTGRES_DSN=postgres://signoz:signoz@signoz-metastore-postgres-0:5432/signoz?sslmode=disable + - SIGNOZ_SQLSTORE_PROVIDER=postgres + - SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN=tcp://signoz-telemetrystore-clickhouse-0-0:9000 + - SIGNOZ_TELEMETRYSTORE_PROVIDER=clickhouse + healthcheck: + interval: 30s + retries: 3 + test: + - CMD + - wget + - --spider + - -q + - localhost:8080/api/v1/health + timeout: 5s + image: signoz/signoz:latest + logging: + options: + max-file: "3" + max-size: 50m + volumes: + - source: signoz-signoz-0-data + target: /var/lib/signoz/ + type: volume + signoz-telemetrykeeper-clickhousekeeper-0: + entrypoint: + - /usr/bin/clickhouse-keeper + - --config-file=/etc/clickhouse-keeper/keeper.yaml + healthcheck: + interval: 30s + retries: 3 + start_period: 40s + test: + - CMD + - clickhouse-keeper-client + - -h + - localhost + - -p + - "9181" + - -q + - ls + timeout: 10s + image: clickhouse/clickhouse-keeper:25.5.6 + logging: + options: + max-file: "3" + max-size: 50m + volumes: + - source: signoz-telemetrykeeper-0-data + target: /var/lib/clickhouse-keeper + type: volume + - content: | + listen_host: 0.0.0.0 + logger: + level: information + console: true + keeper_server: + four_letter_word_white_list: "*" + coordination_settings: + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: signoz-telemetrykeeper-clickhousekeeper-0 + port: 9234 + id: 0 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 + source: ./configs/telemetrykeeper/keeper-0.yaml + target: /etc/clickhouse-keeper/keeper.yaml + type: bind + signoz-telemetrystore-clickhouse-0-0: + depends_on: + signoz-telemetrykeeper-clickhousekeeper-0: + condition: service_healthy + signoz-telemetrystore-clickhouse-user-scripts: + condition: service_completed_successfully + environment: + - CLICKHOUSE_SKIP_USER_SETUP=1 + - CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.yaml + healthcheck: + interval: 30s + retries: 3 + test: + - CMD + - wget + - --spider + - -q + - 0.0.0.0:8123/ping + timeout: 5s + image: clickhouse/clickhouse-server:25.5.6 + logging: + options: + max-file: "3" + max-size: 50m + volumes: + - source: signoz-telemetrystore-0-0-data + target: /var/lib/clickhouse/ + type: volume + - read_only: true + source: signoz-telemetrystore-user-scripts + target: /var/lib/clickhouse/user_scripts + type: volume + - content: | + path: /var/lib/clickhouse/ + tmp_path: /var/lib/clickhouse/tmp/ + user_files_path: /var/lib/clickhouse/user_files/ + format_schema_path: /var/lib/clickhouse/format_schemas/ + dictionaries_config: '*_dictionary.xml' + display_name: cluster + distributed_ddl: + path: /clickhouse/task_queue/ddl + http_port: 8123 + interserver_http_port: 9009 + listen_host: 0.0.0.0 + logger: + console: 1 + count: 10 + formatting: + type: console + level: information + size: 1000M + macros: + replica: "00" + shard: "00" + profiles: + default: + allow_simdjson: 0 + load_balancing: random + log_queries: 1 + quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 + user_directories: + users_xml: + path: users.xml + remote_servers: + cluster: + shard: + - replica: + - host: signoz-telemetrystore-clickhouse-0-0 + port: 9000 + zookeeper: + node: + - host: signoz-telemetrykeeper-clickhousekeeper-0 + port: 9181 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_thread_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_views_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + part_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + metric_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + asynchronous_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + trace_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + error_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + latency_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + session_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + text_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + zookeeper_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + tcp_port: 9000 + user_defined_executable_functions_config: '*function.yaml' + user_scripts_path: /var/lib/clickhouse/user_scripts/ + users: + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 + source: ./configs/telemetrystore/config-0-0.yaml + target: /etc/clickhouse-server/config.yaml + type: bind + - content: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable + source: ./configs/telemetrystore/functions.yaml + target: /etc/clickhouse-server/functions.yaml + type: bind + signoz-telemetrystore-clickhouse-user-scripts: + command: + - bash + - -c + - | + version="v0.0.1" + node_os=$$(uname -s | tr '[:upper:]' '[:lower:]') + node_arch=$$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) + echo "Fetching histogram-binary for $${node_os}/$${node_arch}" + cd /tmp + wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F$${version}/histogram-quantile_$${node_os}_$${node_arch}.tar.gz" + tar -xvzf histogram-quantile.tar.gz + mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile + exclude_from_hc: true + image: clickhouse/clickhouse-server:25.5.6 + logging: + options: + max-file: "3" + max-size: 50m + restart: "no" + volumes: + - source: signoz-telemetrystore-user-scripts + target: /var/lib/clickhouse/user_scripts + type: volume + signoz-telemetrystore-migrator: + command: + - /signoz-otel-collector migrate bootstrap --clickhouse-dsn tcp://signoz-telemetrystore-clickhouse-0-0:9000 + - /signoz-otel-collector migrate sync up --clickhouse-dsn tcp://signoz-telemetrystore-clickhouse-0-0:9000 + - /signoz-otel-collector migrate async up --clickhouse-dsn tcp://signoz-telemetrystore-clickhouse-0-0:9000 + depends_on: + signoz-telemetrystore-clickhouse-0-0: + condition: service_healthy + entrypoint: + - /bin/sh + - -c + environment: + - SIGNOZ_OTEL_COLLECTOR_TIMEOUT=10m + exclude_from_hc: true + image: signoz/signoz-otel-collector:latest + logging: + options: + max-file: "3" + max-size: 50m + restart: on-failure +volumes: + signoz-metastore-0-data: + name: signoz-metastore-0-data + signoz-signoz-0-data: + name: signoz-signoz-0-data + signoz-telemetrykeeper-0-data: + name: signoz-telemetrykeeper-0-data + signoz-telemetrystore-0-0-data: + name: signoz-telemetrystore-0-0-data + signoz-telemetrystore-user-scripts: + name: signoz-telemetrystore-user-scripts diff --git a/docs/examples/docker/README.md b/docs/examples/docker/README.md new file mode 100644 index 0000000..592c01c --- /dev/null +++ b/docs/examples/docker/README.md @@ -0,0 +1,6 @@ +# Docker + +| Flavor | Description | +| --- | --- | +| [compose](compose/) | Single-node Docker Compose deployment | +| [swarm](swarm/) | Multi-node Docker Swarm cluster | diff --git a/docs/examples/docker/compose/README.md b/docs/examples/docker/compose/README.md new file mode 100644 index 0000000..22e8b5a --- /dev/null +++ b/docs/examples/docker/compose/README.md @@ -0,0 +1,85 @@ +# Docker Compose + +| Field | Value | +| --- | --- | +| **Mode** | `docker` | +| **Flavor** | `compose` | +| **Platform** | `-` | + +## Overview + +Deploys SigNoz using Docker Compose with all Foundry defaults. This is the simplest way to run SigNoz locally or on a single node. + +## Prerequisites + +- Docker Engine 20.10+ +- Docker Compose v2 + +## Configuration + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + flavor: compose + mode: docker +``` + +## Deploy + +```bash +foundryctl cast -f casting.yaml +``` + +Or step by step: + +```bash +# Validate prerequisites +foundryctl gauge -f casting.yaml + +# Generate compose files +foundryctl forge -f casting.yaml + +# Start the stack +cd pours/deployment && docker compose up -d +``` + +## Generated output + +```text +pours/deployment/ + compose.yaml + configs/ + ingester/ + ingester.yaml + opamp.yaml + telemetrykeeper/ + clickhousekeeper/ + keeper-0.yaml + telemetrystore/ + clickhouse/ + config.yaml + functions.yaml +``` + +## After deployment + +```bash +# Check running containers +docker ps + +# View logs for a specific service +docker compose -f pours/deployment/compose.yaml logs -f signoz + +# Stop the stack +cd pours/deployment && docker compose down +``` + +> [!NOTE] +> - `foundryctl cast` detects whether `docker compose` (v2 plugin) or `docker-compose` (legacy standalone) is available and uses whichever it finds, preferring the v2 plugin. + +## Customization + +Override component images, replicas, or environment variables in the casting spec. For platform-level changes to the generated `compose.yaml` (memory limits, networks, volumes), use [patches](../../../concepts/patches.md). diff --git a/docs/examples/docker/compose/casting.yaml b/docs/examples/docker/compose/casting.yaml index 2f57ae0..e05f7f0 100644 --- a/docs/examples/docker/compose/casting.yaml +++ b/docs/examples/docker/compose/casting.yaml @@ -1,4 +1,5 @@ apiVersion: v1alpha1 +kind: Installation metadata: name: o11y spec: diff --git a/docs/examples/docker/compose/casting.yaml.lock b/docs/examples/docker/compose/casting.yaml.lock index f0fbeed..d934011 100644 --- a/docs/examples/docker/compose/casting.yaml.lock +++ b/docs/examples/docker/compose/casting.yaml.lock @@ -1,4 +1,5 @@ apiVersion: v1alpha1 +kind: Installation metadata: name: o11y spec: @@ -28,9 +29,9 @@ spec: endpoint: "0.0.0.0:4318" processors: batch: - send_batch_size: 20000 - send_batch_max_size: 25000 - timeout: 1s + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s batch/meter: send_batch_size: 20000 send_batch_max_size: 25000 @@ -90,6 +91,12 @@ spec: timeout: 45s sending_queue: enabled: false + metadataexporter: + enabled: true + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory extensions: o11y_health_check: endpoint: "0.0.0.0:13133" @@ -163,9 +170,9 @@ spec: endpoint: "0.0.0.0:4318" processors: batch: - send_batch_size: 20000 - send_batch_max_size: 25000 - timeout: 1s + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s batch/meter: send_batch_size: 20000 send_batch_max_size: 25000 @@ -225,6 +232,12 @@ spec: timeout: 45s sending_queue: enabled: false + metadataexporter: + enabled: true + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory extensions: o11y_health_check: endpoint: "0.0.0.0:13133" @@ -337,6 +350,9 @@ spec: operation_timeout_ms: 10000 raft_logs_level: warning session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 log_storage_path: /var/lib/clickhouse/coordination/log raft_configuration: server: @@ -368,6 +384,9 @@ spec: operation_timeout_ms: 10000 raft_logs_level: warning session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 log_storage_path: /var/lib/clickhouse/coordination/log raft_configuration: server: @@ -385,7 +404,7 @@ spec: shards: 1 config: data: - config.yaml: | + config-0-0.yaml: | path: /var/lib/clickhouse/ tmp_path: /var/lib/clickhouse/tmp/ user_files_path: /var/lib/clickhouse/user_files/ @@ -405,13 +424,13 @@ spec: level: information size: 1000M macros: - replica: "01" - shard: "01" + replica: "00" + shard: "00" profiles: default: + allow_simdjson: 0 load_balancing: random log_queries: 1 - max_memory_usage: 10000000000 quotas: default: interval: @@ -434,6 +453,40 @@ spec: node: - host: o11y-telemetrykeeper-clickhousekeeper-0 port: 9181 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_thread_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_views_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + part_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + metric_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + asynchronous_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + trace_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + error_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + latency_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + session_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + text_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + zookeeper_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" tcp_port: 9000 user_defined_executable_functions_config: '*function.yaml' user_scripts_path: /var/lib/clickhouse/user_scripts/ @@ -471,7 +524,7 @@ spec: - tcp://o11y-telemetrystore-clickhouse-0-0:9000 config: data: - config.yaml: | + config-0-0.yaml: | path: /var/lib/clickhouse/ tmp_path: /var/lib/clickhouse/tmp/ user_files_path: /var/lib/clickhouse/user_files/ @@ -491,13 +544,13 @@ spec: level: information size: 1000M macros: - replica: "01" - shard: "01" + replica: "00" + shard: "00" profiles: default: + allow_simdjson: 0 load_balancing: random log_queries: 1 - max_memory_usage: 10000000000 quotas: default: interval: @@ -520,6 +573,40 @@ spec: node: - host: o11y-telemetrykeeper-clickhousekeeper-0 port: 9181 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_thread_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_views_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + part_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + metric_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + asynchronous_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + trace_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + error_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + latency_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + session_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + text_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + zookeeper_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" tcp_port: 9000 user_defined_executable_functions_config: '*function.yaml' user_scripts_path: /var/lib/clickhouse/user_scripts/ diff --git a/docs/examples/docker/compose/pours/deployment/compose.yaml b/docs/examples/docker/compose/pours/deployment/compose.yaml index 55c11a2..59517b9 100644 --- a/docs/examples/docker/compose/pours/deployment/compose.yaml +++ b/docs/examples/docker/compose/pours/deployment/compose.yaml @@ -1,3 +1,4 @@ +name: signoz networks: o11y-network: name: o11y-network @@ -45,7 +46,7 @@ services: - CMD-SHELL - pg_isready -U o11y -d o11y timeout: 10s - image: docker.io/postgres:16 + image: postgres:16 networks: - o11y-network volumes: @@ -57,7 +58,8 @@ services: o11y-telemetrystore-clickhouse-0-0: condition: service_healthy deploy: - replicas: 1 + restart_policy: + condition: on-failure environment: - HANZO_ALERTMANAGER_PROVIDER=o11y - HANZO_TELEMETRYSTORE_CLICKHOUSE_DSN=tcp://o11y-telemetrystore-clickhouse-0-0:9000 @@ -77,7 +79,7 @@ services: - wget - --spider - -q - - http://localhost:3301/api/v1/health + - http://localhost:8080/api/v1/health timeout: 10s image: ghcr.io/hanzoai/o11y:latest networks: diff --git a/docs/examples/docker/compose/pours/deployment/configs/ingester/opamp.yaml b/docs/examples/docker/compose/pours/deployment/configs/ingester/opamp.yaml deleted file mode 100644 index 4073483..0000000 --- a/docs/examples/docker/compose/pours/deployment/configs/ingester/opamp.yaml +++ /dev/null @@ -1 +0,0 @@ -server_endpoint: ws://o11y-o11y:4320/v1/opamp diff --git a/docs/examples/docker/compose/pours/deployment/configs/ingester/ingester.yaml b/docs/examples/docker/compose/pours/deployment/ingester/ingester.yaml similarity index 97% rename from docs/examples/docker/compose/pours/deployment/configs/ingester/ingester.yaml rename to docs/examples/docker/compose/pours/deployment/ingester/ingester.yaml index f7df397..4cfa3da 100644 --- a/docs/examples/docker/compose/pours/deployment/configs/ingester/ingester.yaml +++ b/docs/examples/docker/compose/pours/deployment/ingester/ingester.yaml @@ -36,9 +36,9 @@ extensions: endpoint: 0.0.0.0:13133 processors: batch: - send_batch_max_size: 25000 - send_batch_size: 20000 - timeout: 1s + send_batch_max_size: 55000 + send_batch_size: 50000 + timeout: 5s batch/meter: send_batch_max_size: 25000 send_batch_size: 20000 diff --git a/docs/examples/docker/compose/pours/deployment/ingester/opamp.yaml b/docs/examples/docker/compose/pours/deployment/ingester/opamp.yaml new file mode 100644 index 0000000..2672ddb --- /dev/null +++ b/docs/examples/docker/compose/pours/deployment/ingester/opamp.yaml @@ -0,0 +1 @@ +server_endpoint: ws://signoz-signoz-0:4320/v1/opamp diff --git a/docs/examples/docker/compose/pours/deployment/telemetrykeeper/clickhousekeeper/keeper-0.yaml b/docs/examples/docker/compose/pours/deployment/telemetrykeeper/clickhousekeeper/keeper-0.yaml new file mode 100644 index 0000000..5c1e6fc --- /dev/null +++ b/docs/examples/docker/compose/pours/deployment/telemetrykeeper/clickhousekeeper/keeper-0.yaml @@ -0,0 +1,22 @@ +keeper_server: + coordination_settings: + force_sync: false + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + snapshot_distance: 100000 + snapshots_to_keep: 3 + four_letter_word_white_list: '*' + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: signoz-telemetrykeeper-clickhousekeeper-0 + id: 0 + port: 9234 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 +listen_host: 0.0.0.0 +logger: + console: true + level: information diff --git a/docs/examples/docker/compose/pours/deployment/configs/telemetrystore/config.yaml b/docs/examples/docker/compose/pours/deployment/telemetrystore/clickhouse/config-0-0.yaml similarity index 54% rename from docs/examples/docker/compose/pours/deployment/configs/telemetrystore/config.yaml rename to docs/examples/docker/compose/pours/deployment/telemetrystore/clickhouse/config-0-0.yaml index 38bdaff..8e9ac45 100644 --- a/docs/examples/docker/compose/pours/deployment/configs/telemetrystore/config.yaml +++ b/docs/examples/docker/compose/pours/deployment/telemetrystore/clickhouse/config-0-0.yaml @@ -1,10 +1,16 @@ +asynchronous_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE dictionaries_config: '*_dictionary.xml' display_name: cluster distributed_ddl: path: /clickhouse/task_queue/ddl +error_log: + ttl: event_date + INTERVAL 1 DAY DELETE format_schema_path: /var/lib/clickhouse/format_schemas/ http_port: 8123 interserver_http_port: 9009 +latency_log: + ttl: event_date + INTERVAL 1 DAY DELETE listen_host: 0.0.0.0 logger: console: 1 @@ -14,14 +20,32 @@ logger: level: information size: 1000M macros: - replica: "01" - shard: "01" + replica: "00" + shard: "00" +metric_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE +part_log: + ttl: event_date + INTERVAL 1 DAY DELETE path: /var/lib/clickhouse/ +processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE profiles: default: + allow_simdjson: 0 load_balancing: random log_queries: 1 - max_memory_usage: 10000000000 +query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: event_date + INTERVAL 1 DAY DELETE +query_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE +query_thread_log: + ttl: event_date + INTERVAL 1 DAY DELETE +query_views_log: + ttl: event_date + INTERVAL 1 DAY DELETE quotas: default: interval: @@ -37,8 +61,16 @@ remote_servers: - replica: host: o11y-telemetrystore-clickhouse-0-0 port: 9000 +session_log: + ttl: event_date + INTERVAL 1 DAY DELETE tcp_port: 9000 +text_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE tmp_path: /var/lib/clickhouse/tmp/ +trace_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE user_defined_executable_functions_config: '*function.yaml' user_directories: users_xml: @@ -60,3 +92,5 @@ zookeeper: node: - host: o11y-telemetrykeeper-clickhousekeeper-0 port: 9181 +zookeeper_log: + ttl: event_date + INTERVAL 1 DAY DELETE diff --git a/docs/examples/docker/compose/pours/deployment/configs/telemetrystore/functions.yaml b/docs/examples/docker/compose/pours/deployment/telemetrystore/clickhouse/functions.yaml similarity index 100% rename from docs/examples/docker/compose/pours/deployment/configs/telemetrystore/functions.yaml rename to docs/examples/docker/compose/pours/deployment/telemetrystore/clickhouse/functions.yaml diff --git a/docs/examples/docker/swarm/README.md b/docs/examples/docker/swarm/README.md new file mode 100644 index 0000000..e81b5a4 --- /dev/null +++ b/docs/examples/docker/swarm/README.md @@ -0,0 +1,79 @@ +# Docker Swarm + +| Field | Value | +| --- | --- | +| **Mode** | `docker` | +| **Flavor** | `swarm` | +| **Platform** | `-` | + +## Overview + +Deploys SigNoz on a Docker Swarm cluster. Foundry generates a Compose file and deploys it as a stack using `docker stack deploy`. + +## Prerequisites + +- Docker Engine 20.10+ with Swarm mode initialized (`docker swarm init`) +- At least one manager node + +## Configuration + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + flavor: swarm + mode: docker +``` + +## Deploy + +```bash +foundryctl cast -f casting.yaml +``` + +Or step by step: + +```bash +# Generate the compose file +foundryctl forge -f casting.yaml + +# Deploy manually +docker stack deploy -c pours/deployment/compose.yaml signoz +``` + +## Generated output + +```text +pours/deployment/ + compose.yaml + configs/ + ingester/ + ingester.yaml + opamp.yaml + telemetrykeeper/ + clickhousekeeper/ + keeper-0.yaml + telemetrystore/ + clickhouse/ + config.yaml + functions.yaml +``` + +## After deployment + +```bash +# List services in the stack +docker stack services signoz + +# View logs for a service +docker service logs signoz_signoz -f + +# Remove the stack +docker stack rm signoz +``` + +## Customization + +For platform-level changes to the generated `compose.yaml`, use [patches](../../../concepts/patches.md). diff --git a/docs/examples/docker/swarm/casting.yaml b/docs/examples/docker/swarm/casting.yaml new file mode 100644 index 0000000..5fb97fb --- /dev/null +++ b/docs/examples/docker/swarm/casting.yaml @@ -0,0 +1,8 @@ +apiVersion: v1alpha1 +kind: Installation +metadata: + name: signoz +spec: + deployment: + flavor: swarm + mode: docker diff --git a/docs/examples/docker/swarm/casting.yaml.lock b/docs/examples/docker/swarm/casting.yaml.lock new file mode 100644 index 0000000..0f633e4 --- /dev/null +++ b/docs/examples/docker/swarm/casting.yaml.lock @@ -0,0 +1,648 @@ +apiVersion: v1alpha1 +kind: Installation +metadata: + name: signoz +spec: + deployment: + flavor: swarm + mode: docker + ingester: + spec: + cluster: + replicas: 1 + config: + data: + ingester.yaml: | + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 5s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + metadataexporter: + enabled: true + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + - metadataexporter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + - metadataexporter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + - metadataexporter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: | + server_endpoint: ws://signoz-signoz-0:4320/v1/opamp + enabled: true + env: + SIGNOZ_OTEL_COLLECTOR_TIMEOUT: 10m + image: signoz/signoz-otel-collector:latest + version: latest + status: + addresses: + otlp: + - tcp://signoz-ingester:4318 + - tcp://signoz-ingester:4317 + config: + data: + ingester.yaml: | + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 5s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + metadataexporter: + enabled: true + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + - metadataexporter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + - metadataexporter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + - metadataexporter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: | + server_endpoint: ws://signoz-signoz-0:4320/v1/opamp + env: + SIGNOZ_OTEL_COLLECTOR_TIMEOUT: 10m + metastore: + kind: postgres + spec: + cluster: + replicas: 1 + config: {} + enabled: true + env: + POSTGRES_DB: signoz + POSTGRES_PASSWORD: signoz + POSTGRES_USER: signoz + image: postgres:16 + version: "16" + status: + addresses: + dsn: + - tcp://signoz-metastore-postgres-0:5432 + config: {} + env: + POSTGRES_DB: signoz + POSTGRES_PASSWORD: signoz + POSTGRES_USER: signoz + signoz: + spec: + cluster: + replicas: 1 + config: {} + enabled: true + env: + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@signoz-metastore-postgres-0:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://signoz-telemetrystore-clickhouse-0-0:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + image: signoz/signoz:latest + version: latest + status: + addresses: + apiserver: + - tcp://signoz-signoz-0:8080 + opamp: + - ws://signoz-signoz-0:4320 + config: {} + env: + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@signoz-metastore-postgres-0:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://signoz-telemetrystore-clickhouse-0-0:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + telemetrykeeper: + kind: clickhousekeeper + spec: + cluster: + replicas: 1 + config: + data: + keeper-0.yaml: | + listen_host: 0.0.0.0 + logger: + level: information + console: true + keeper_server: + four_letter_word_white_list: "*" + coordination_settings: + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: signoz-telemetrykeeper-clickhousekeeper-0 + port: 9234 + id: 0 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 + enabled: true + image: clickhouse/clickhouse-keeper:25.5.6 + version: 25.5.6 + status: + addresses: + client: + - tcp://signoz-telemetrykeeper-clickhousekeeper-0:9181 + raft: + - tcp://signoz-telemetrykeeper-clickhousekeeper-0:9234 + config: + data: + keeper-0.yaml: | + listen_host: 0.0.0.0 + logger: + level: information + console: true + keeper_server: + four_letter_word_white_list: "*" + coordination_settings: + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: signoz-telemetrykeeper-clickhousekeeper-0 + port: 9234 + id: 0 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 + telemetrystore: + kind: clickhouse + spec: + cluster: + replicas: 0 + shards: 1 + config: + data: + config-0-0.yaml: | + path: /var/lib/clickhouse/ + tmp_path: /var/lib/clickhouse/tmp/ + user_files_path: /var/lib/clickhouse/user_files/ + format_schema_path: /var/lib/clickhouse/format_schemas/ + dictionaries_config: '*_dictionary.xml' + display_name: cluster + distributed_ddl: + path: /clickhouse/task_queue/ddl + http_port: 8123 + interserver_http_port: 9009 + listen_host: 0.0.0.0 + logger: + console: 1 + count: 10 + formatting: + type: console + level: information + size: 1000M + macros: + replica: "00" + shard: "00" + profiles: + default: + allow_simdjson: 0 + load_balancing: random + log_queries: 1 + quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 + user_directories: + users_xml: + path: users.xml + remote_servers: + cluster: + shard: + - replica: + - host: signoz-telemetrystore-clickhouse-0-0 + port: 9000 + zookeeper: + node: + - host: signoz-telemetrykeeper-clickhousekeeper-0 + port: 9181 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_thread_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_views_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + part_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + metric_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + asynchronous_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + trace_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + error_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + latency_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + session_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + text_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + zookeeper_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + tcp_port: 9000 + user_defined_executable_functions_config: '*function.yaml' + user_scripts_path: /var/lib/clickhouse/user_scripts/ + users: + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 + functions.yaml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable + enabled: true + image: clickhouse/clickhouse-server:25.5.6 + version: 25.5.6 + status: + addresses: + tcp: + - tcp://signoz-telemetrystore-clickhouse-0-0:9000 + config: + data: + config-0-0.yaml: | + path: /var/lib/clickhouse/ + tmp_path: /var/lib/clickhouse/tmp/ + user_files_path: /var/lib/clickhouse/user_files/ + format_schema_path: /var/lib/clickhouse/format_schemas/ + dictionaries_config: '*_dictionary.xml' + display_name: cluster + distributed_ddl: + path: /clickhouse/task_queue/ddl + http_port: 8123 + interserver_http_port: 9009 + listen_host: 0.0.0.0 + logger: + console: 1 + count: 10 + formatting: + type: console + level: information + size: 1000M + macros: + replica: "00" + shard: "00" + profiles: + default: + allow_simdjson: 0 + load_balancing: random + log_queries: 1 + quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 + user_directories: + users_xml: + path: users.xml + remote_servers: + cluster: + shard: + - replica: + - host: signoz-telemetrystore-clickhouse-0-0 + port: 9000 + zookeeper: + node: + - host: signoz-telemetrykeeper-clickhousekeeper-0 + port: 9181 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_thread_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_views_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + part_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + metric_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + asynchronous_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + trace_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + error_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + latency_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + session_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + text_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + zookeeper_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + tcp_port: 9000 + user_defined_executable_functions_config: '*function.yaml' + user_scripts_path: /var/lib/clickhouse/user_scripts/ + users: + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 + functions.yaml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable diff --git a/docs/examples/docker/swarm/pours/deployment/compose.yaml b/docs/examples/docker/swarm/pours/deployment/compose.yaml new file mode 100644 index 0000000..f8926ee --- /dev/null +++ b/docs/examples/docker/swarm/pours/deployment/compose.yaml @@ -0,0 +1,197 @@ +configs: + ingester-config: + file: ingester/ingester.yaml + manager-config: + file: ingester/opamp.yaml + telemetrykeeper-0: + file: telemetrykeeper/clickhousekeeper/keeper-0.yaml + telemetrystore-config-0-0: + file: telemetrystore/clickhouse/config-0-0.yaml + telemetrystore-functions: + file: telemetrystore/clickhouse/functions.yaml +networks: + signoz-network: + attachable: true + driver: overlay + name: signoz-network +services: + ingester: + command: + - -c + - | + /signoz-otel-collector migrate sync check && + /signoz-otel-collector --config=/etc/otel-collector-config.yaml --manager-config=/etc/opamp-config.yaml --copy-path=/var/tmp/collector-config.yaml + configs: + - source: ingester-config + target: /etc/otel-collector-config.yaml + - source: manager-config + target: /etc/opamp-config.yaml + deploy: + replicas: 1 + entrypoint: + - /bin/sh + environment: + - SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN=tcp://signoz-telemetrystore-clickhouse-0-0:9000 + - SIGNOZ_OTEL_COLLECTOR_TIMEOUT=10m + image: signoz/signoz-otel-collector:latest + networks: + signoz-network: + aliases: + - signoz-ingester + ports: + - 4317:4317 + - 4318:4318 + signoz-metastore-postgres-0: + deploy: + restart_policy: + condition: on-failure + environment: + - POSTGRES_DB=signoz + - POSTGRES_PASSWORD=signoz + - POSTGRES_USER=signoz + healthcheck: + interval: 30s + retries: 3 + start_period: 30s + test: + - CMD-SHELL + - pg_isready -U signoz -d signoz + timeout: 10s + image: postgres:16 + networks: + - signoz-network + volumes: + - signoz-metastore-postgres-0-data:/var/lib/postgresql/data + signoz-signoz-0: + deploy: + restart_policy: + condition: on-failure + environment: + - SIGNOZ_SQLSTORE_POSTGRES_DSN=postgres://signoz:signoz@signoz-metastore-postgres-0:5432/signoz?sslmode=disable + - SIGNOZ_SQLSTORE_PROVIDER=postgres + - SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN=tcp://signoz-telemetrystore-clickhouse-0-0:9000 + - SIGNOZ_TELEMETRYSTORE_PROVIDER=clickhouse + healthcheck: + interval: 30s + retries: 3 + start_period: 60s + test: + - CMD + - wget + - --spider + - -q + - http://localhost:8080/api/v1/health + timeout: 10s + image: signoz/signoz:latest + networks: + - signoz-network + ports: + - 8080:8080 + signoz-telemetrykeeper-clickhousekeeper-0: + configs: + - source: telemetrykeeper-0 + target: /etc/clickhouse-keeper/keeper.yaml + deploy: + restart_policy: + condition: on-failure + entrypoint: + - /usr/bin/clickhouse-keeper + - --config-file=/etc/clickhouse-keeper/keeper.yaml + healthcheck: + interval: 30s + retries: 3 + start_period: 40s + test: + - CMD + - clickhouse-keeper-client + - -h + - localhost + - -p + - "9181" + - -q + - ls + timeout: 10s + image: clickhouse/clickhouse-keeper:25.5.6 + networks: + - signoz-network + volumes: + - signoz-telemetrykeeper-0-data:/var/lib/clickhouse-keeper + signoz-telemetrystore-clickhouse-0-0: + configs: + - source: telemetrystore-config-0-0 + target: /etc/clickhouse-server/config.yaml + - source: telemetrystore-functions + target: /etc/clickhouse-server/functions.yaml + deploy: + restart_policy: + condition: on-failure + environment: + - CLICKHOUSE_SKIP_USER_SETUP=1 + - CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.yaml + healthcheck: + interval: 30s + retries: 3 + start_period: 40s + test: + - CMD + - wget + - --spider + - -q + - http://localhost:8123/ping + timeout: 10s + hostname: signoz-telemetrystore-clickhouse-0-0 + image: clickhouse/clickhouse-server:25.5.6 + networks: + - signoz-network + volumes: + - signoz-telemetrystore-0-0-data:/var/lib/clickhouse + - signoz-telemetrystore-user-scripts:/var/lib/clickhouse/user_scripts:ro + signoz-telemetrystore-clickhouse-user-scripts: + command: + - bash + - -c + - |- + version="v0.0.1" + node_os=$$(uname -s | tr '[:upper:]' '[:lower:]') + node_arch=$$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) + echo "Fetching histogram-binary for $${node_os}/$${node_arch}" + cd /tmp + wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F$${version}/histogram-quantile_$${node_os}_$${node_arch}.tar.gz" + tar -xvzf histogram-quantile.tar.gz + mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile + deploy: + mode: global + restart_policy: + condition: none + image: clickhouse/clickhouse-server:25.5.6 + networks: + - signoz-network + volumes: + - signoz-telemetrystore-user-scripts:/var/lib/clickhouse/user_scripts + signoz-telemetrystore-migrator: + command: + - -c + - | + /signoz-otel-collector migrate ready && + /signoz-otel-collector migrate bootstrap && + /signoz-otel-collector migrate sync up && + /signoz-otel-collector migrate async up + deploy: + replicas: 1 + restart_policy: + condition: on-failure + max_attempts: 3 + entrypoint: + - /bin/sh + environment: + - SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN=tcp://signoz-telemetrystore-clickhouse-0-0:9000 + - SIGNOZ_OTEL_COLLECTOR_TIMEOUT=10m + image: signoz/signoz-otel-collector:latest + networks: + - signoz-network +version: "3" +volumes: + signoz-metastore-postgres-0-data: null + signoz-telemetrykeeper-0-data: null + signoz-telemetrystore-0-0-data: null + signoz-telemetrystore-user-scripts: null diff --git a/docs/examples/docker/swarm/pours/deployment/ingester/ingester.yaml b/docs/examples/docker/swarm/pours/deployment/ingester/ingester.yaml new file mode 100644 index 0000000..44e476e --- /dev/null +++ b/docs/examples/docker/swarm/pours/deployment/ingester/ingester.yaml @@ -0,0 +1,132 @@ +connectors: + signozmeter: + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + metrics_flush_interval: 1h +exporters: + clickhouselogsexporter: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_logs + sending_queue: + enabled: false + timeout: 45s + use_new_schema: true + clickhousetraces: + datasource: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + sending_queue: + enabled: false + timeout: 45s + use_new_schema: true + metadataexporter: + cache: + provider: in_memory + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_metadata + enabled: true + timeout: 45s + signozclickhousemeter: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_meter + sending_queue: + enabled: false + timeout: 45s + signozclickhousemetrics: + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_metrics + sending_queue: + enabled: false + timeout: 45s +extensions: + pprof: + endpoint: 0.0.0.0:1777 + signoz_health_check: + endpoint: 0.0.0.0:13133 +processors: + batch: + send_batch_max_size: 55000 + send_batch_size: 50000 + timeout: 5s + batch/meter: + send_batch_max_size: 25000 + send_batch_size: 20000 + timeout: 5s + signozspanmetrics/delta: + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + dimensions: + - default: default + name: service.namespace + - default: default + name: deployment.environment + - name: signoz.collector.id + - name: service.version + dimensions_cache_size: 100000 + enable_exp_histogram: true + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +service: + extensions: + - signoz_health_check + - pprof + pipelines: + logs: + exporters: + - clickhouselogsexporter + - signozmeter + - metadataexporter + processors: + - batch + receivers: + - otlp + metrics: + exporters: + - signozclickhousemetrics + - signozmeter + - metadataexporter + processors: + - batch + receivers: + - otlp + metrics/meter: + exporters: + - signozclickhousemeter + processors: + - batch/meter + receivers: + - signozmeter + traces: + exporters: + - clickhousetraces + - signozmeter + - metadataexporter + processors: + - signozspanmetrics/delta + - batch + receivers: + - otlp + telemetry: + logs: + encoding: json diff --git a/docs/examples/docker/swarm/pours/deployment/ingester/opamp.yaml b/docs/examples/docker/swarm/pours/deployment/ingester/opamp.yaml new file mode 100644 index 0000000..2672ddb --- /dev/null +++ b/docs/examples/docker/swarm/pours/deployment/ingester/opamp.yaml @@ -0,0 +1 @@ +server_endpoint: ws://signoz-signoz-0:4320/v1/opamp diff --git a/docs/examples/docker/compose/pours/deployment/configs/telemetrykeeper/keeper-0.yaml b/docs/examples/docker/swarm/pours/deployment/telemetrykeeper/clickhousekeeper/keeper-0.yaml similarity index 86% rename from docs/examples/docker/compose/pours/deployment/configs/telemetrykeeper/keeper-0.yaml rename to docs/examples/docker/swarm/pours/deployment/telemetrykeeper/clickhousekeeper/keeper-0.yaml index e1ca33d..1606da9 100644 --- a/docs/examples/docker/compose/pours/deployment/configs/telemetrykeeper/keeper-0.yaml +++ b/docs/examples/docker/swarm/pours/deployment/telemetrykeeper/clickhousekeeper/keeper-0.yaml @@ -1,8 +1,11 @@ keeper_server: coordination_settings: + force_sync: false operation_timeout_ms: 10000 raft_logs_level: warning session_timeout_ms: 30000 + snapshot_distance: 100000 + snapshots_to_keep: 3 four_letter_word_white_list: '*' log_storage_path: /var/lib/clickhouse/coordination/log raft_configuration: diff --git a/docs/examples/docker/swarm/pours/deployment/telemetrystore/clickhouse/config-0-0.yaml b/docs/examples/docker/swarm/pours/deployment/telemetrystore/clickhouse/config-0-0.yaml new file mode 100644 index 0000000..34d42ff --- /dev/null +++ b/docs/examples/docker/swarm/pours/deployment/telemetrystore/clickhouse/config-0-0.yaml @@ -0,0 +1,96 @@ +asynchronous_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE +dictionaries_config: '*_dictionary.xml' +display_name: cluster +distributed_ddl: + path: /clickhouse/task_queue/ddl +error_log: + ttl: event_date + INTERVAL 1 DAY DELETE +format_schema_path: /var/lib/clickhouse/format_schemas/ +http_port: 8123 +interserver_http_port: 9009 +latency_log: + ttl: event_date + INTERVAL 1 DAY DELETE +listen_host: 0.0.0.0 +logger: + console: 1 + count: 10 + formatting: + type: console + level: information + size: 1000M +macros: + replica: "00" + shard: "00" +metric_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE +part_log: + ttl: event_date + INTERVAL 1 DAY DELETE +path: /var/lib/clickhouse/ +processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE +profiles: + default: + allow_simdjson: 0 + load_balancing: random + log_queries: 1 +query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: event_date + INTERVAL 1 DAY DELETE +query_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE +query_thread_log: + ttl: event_date + INTERVAL 1 DAY DELETE +query_views_log: + ttl: event_date + INTERVAL 1 DAY DELETE +quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 +remote_servers: + cluster: + shard: + - replica: + - host: signoz-telemetrystore-clickhouse-0-0 + port: 9000 +session_log: + ttl: event_date + INTERVAL 1 DAY DELETE +tcp_port: 9000 +text_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE +tmp_path: /var/lib/clickhouse/tmp/ +trace_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE +user_defined_executable_functions_config: '*function.yaml' +user_directories: + users_xml: + path: users.xml +user_files_path: /var/lib/clickhouse/user_files/ +user_scripts_path: /var/lib/clickhouse/user_scripts/ +users: + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 +zookeeper: + node: + - host: signoz-telemetrykeeper-clickhousekeeper-0 + port: 9181 +zookeeper_log: + ttl: event_date + INTERVAL 1 DAY DELETE diff --git a/docs/examples/systemd/binary/pours/deployment/configs/telemetrystore/functions.yaml b/docs/examples/docker/swarm/pours/deployment/telemetrystore/clickhouse/functions.yaml similarity index 100% rename from docs/examples/systemd/binary/pours/deployment/configs/telemetrystore/functions.yaml rename to docs/examples/docker/swarm/pours/deployment/telemetrystore/clickhouse/functions.yaml diff --git a/docs/examples/ecs/README.md b/docs/examples/ecs/README.md new file mode 100644 index 0000000..64e980b --- /dev/null +++ b/docs/examples/ecs/README.md @@ -0,0 +1,5 @@ +# AWS ECS + +| Mode | Flavor | Description | +| --- | --- | --- | +| [ec2/terraform](ec2/terraform/) | `terraform` | ECS on EC2 with Terraform-managed infrastructure | diff --git a/docs/examples/ecs/ec2/terraform/README.md b/docs/examples/ecs/ec2/terraform/README.md new file mode 100644 index 0000000..8b1a3f1 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/README.md @@ -0,0 +1,266 @@ +# ECS EC2 with Terraform + +| Field | Value | +| --- | --- | +| **Mode** | `ec2` | +| **Flavor** | `terraform` | +| **Platform** | `ecs` | + +## Overview + +Deploys SigNoz on AWS ECS (EC2 launch type) using Terraform. Each component runs as a separate ECS service with AWS Cloud Map for service discovery. + +Components: +- ClickHouse Keeper (telemetry keeper) +- ClickHouse (telemetry store) +- PostgreSQL (metadata store) +- SigNoz (UI + API server on port 8080) +- OTel Collector (ingester) +- Schema migrator (Fargate one-shot task) + +## Prerequisites + +- An existing ECS cluster with an EC2 capacity provider +- A VPC with private subnets +- An S3 bucket for storing component configs +- IAM roles for ECS task and task execution +- [Terraform](https://developer.hashicorp.com/terraform/install) >= 1.0 + +## Configuration + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz + annotations: + foundry.signoz.io/ecs/region: us-east-1 + foundry.signoz.io/ecs/cluster-id: arn:aws:ecs:us-east-1:123456789012:cluster/my-cluster + foundry.signoz.io/ecs/subnet-ids: subnet-abc123,subnet-def456 + foundry.signoz.io/ecs/security-group-ids: sg-abc123 + foundry.signoz.io/ecs/vpc-id: vpc-abc123 + foundry.signoz.io/ecs/config-bucket: my-signoz-configs + foundry.signoz.io/ecs/task-role-arn: arn:aws:iam::123456789012:role/ecs-task-role + foundry.signoz.io/ecs/task-execution-role-arn: arn:aws:iam::123456789012:role/ecs-execution-role + foundry.signoz.io/ecs/capacity-provider: my-capacity-provider +spec: + deployment: + platform: ecs + mode: ec2 + flavor: terraform +``` + +## Deploy + +Run the full pipeline (generate Terraform files and apply): + +```bash +foundryctl cast -f casting.yaml +``` + +> [!NOTE] +> `foundryctl cast` runs `terraform init` followed by `terraform apply -auto-approve`. If you prefer to review the plan before applying, use the step-by-step approach below. + +Step-by-step alternative: + +```bash +# 1. Generate Terraform files +foundryctl forge -f casting.yaml + +# 2. Initialize and apply Terraform +cd pours/deployment +terraform init +terraform apply +``` + +## Generated output + +```text +pours/deployment/ + main.tf.json + variables.tf.json + terraform.tfvars.json + module/ + main.tf.json + variables.tf.json + outputs.tf.json + telemetrykeeper.tf.json + telemetrystore.tf.json + telemetrystore_migrator.tf.json + metastore.tf.json + signoz.tf.json + ingester.tf.json + telemetrykeeper/ + clickhousekeeper/ + keeper-0.yaml + telemetrystore/ + clickhouse/ + config.yaml + functions.yaml + ingester/ + ingester.yaml + opamp.yaml +``` + +## After deployment + +Verify the ECS services are running: + +```bash +aws ecs list-services --cluster my-cluster --region us-east-1 +aws ecs describe-services \ + --cluster my-cluster \ + --services signoz-signoz signoz-ingester signoz-telemetrystore-clickhouse \ + --region us-east-1 +``` + +Check that Cloud Map service discovery is healthy: + +```bash +aws servicediscovery list-services --region us-east-1 +``` + +Access the SigNoz UI by setting up an ALB pointing to the SigNoz service on port 8080. + +## Customization + +The module ships with sensible defaults for CPU and memory. To override them, use `spec.patches` on the generated module files: + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz + annotations: + # ... (same annotations as above) +spec: + deployment: + platform: ecs + mode: ec2 + flavor: terraform + patches: + - target: "deployment/module/signoz.tf.json" + type: jsonpatch + operations: + - op: replace + path: /locals/containers/0/cpu + value: 1024 + - op: replace + path: /locals/containers/0/memory + value: 1024 + - op: replace + path: /locals/containers/0/memoryReservation + value: 1024 + - target: "deployment/module/telemetrystore.tf.json" + type: jsonpatch + operations: + - op: replace + path: /locals/containers/2/cpu + value: 2048 + - op: replace + path: /locals/containers/2/memory + value: 4096 + - op: replace + path: /locals/containers/2/memoryReservation + value: 4096 +``` + +Run `foundryctl forge` to see the generated files and identify the JSON paths you want to patch. + +## Annotations + +Annotations populate `terraform.tfvars.json` so Foundry can generate a ready-to-apply Terraform configuration. + +| Annotation | Maps to tfvar | Description | +| --- | --- | --- | +| `foundry.signoz.io/ecs/region` | `region` | AWS region | +| `foundry.signoz.io/ecs/cluster-id` | `ecs_cluster_id` | ECS cluster ARN or ID | +| `foundry.signoz.io/ecs/subnet-ids` | `subnet_ids` | Comma-separated subnet IDs | +| `foundry.signoz.io/ecs/security-group-ids` | `security_group_ids` | Comma-separated security group IDs | +| `foundry.signoz.io/ecs/vpc-id` | `vpc_id` | VPC ID for Cloud Map namespace | +| `foundry.signoz.io/ecs/config-bucket` | `config_bucket` | S3 bucket for component configs | +| `foundry.signoz.io/ecs/task-role-arn` | `task_role_arn` | IAM role ARN for ECS tasks | +| `foundry.signoz.io/ecs/task-execution-role-arn` | `task_execution_role_arn` | IAM role ARN for task execution | +| `foundry.signoz.io/ecs/capacity-provider` | `capacity_provider` | ECS capacity provider name | + +## Platform details + +### Providers + +| Provider | Version | Purpose | +| --- | --- | --- | +| `hashicorp/aws` | `>= 5.0` | ECS, Cloud Map, S3 | + +### Resources + +The module creates the following AWS resources: + +| Resource | Count | Description | +| --- | --- | --- | +| `aws_service_discovery_private_dns_namespace` | 1 | Cloud Map namespace (`{name}.local`) | +| `aws_ecs_task_definition` | 6 | One per component (including migrator) | +| `aws_ecs_service` | 5 | One per long-running component | +| `aws_service_discovery_service` | 5 | One per long-running component | +| `aws_s3_object` | N | Config files for ClickHouse, Keeper, and Ingester | +| `aws_ecs_task_execution` (data) | 1 | Runs the migrator as a Fargate task | + +### Variables + +| Variable | Type | Description | +| --- | --- | --- | +| `region` | `string` | AWS region | +| `ecs_cluster_id` | `string` | ID of the existing ECS cluster | +| `subnet_ids` | `list(string)` | Subnet IDs for ECS service networking (awsvpc) | +| `security_group_ids` | `list(string)` | Security group IDs for ECS service networking | +| `vpc_id` | `string` | VPC ID for the Cloud Map private DNS namespace | +| `config_bucket` | `string` | S3 bucket for storing component config files | +| `task_role_arn` | `string` | IAM role ARN for ECS tasks | +| `task_execution_role_arn` | `string` | IAM role ARN for ECS task execution (pull images, write logs) | +| `capacity_provider` | `string` | Name of the ECS capacity provider | + +### Outputs + +| Output | Description | +| --- | --- | +| `namespace_id` | Cloud Map private DNS namespace ID | +| `namespace_name` | Cloud Map private DNS namespace name | +| `signoz_service_arn` | SigNoz ECS service ARN (target for ALB on port 8080) | +| `signoz_service_name` | SigNoz ECS service name | +| `ingester_service_arn` | Ingester ECS service ARN (target for NLB on port 4317/4318) | +| `ingester_service_name` | Ingester ECS service name | +| `telemetrystore_service_name` | ClickHouse ECS service name | +| `telemetrykeeper_service_name` | ClickHouse Keeper ECS service name | +| `metastore_service_name` | PostgreSQL ECS service name | + +### Service discovery + +Components communicate via Cloud Map DNS within the `{name}.local` namespace: + +| Component | DNS name | Port | +| --- | --- | --- | +| ClickHouse Keeper | `telemetrykeeper-clickhousekeeper.{name}.local` | 9181 (client), 9234 (raft) | +| ClickHouse | `telemetrystore-clickhouse.{name}.local` | 9000 (native), 8123 (HTTP) | +| PostgreSQL | `metastore-postgresql.{name}.local` | 5432 | +| SigNoz | `signoz.{name}.local` | 8080 (API), 4320 (OpAMP) | +| Ingester | `ingester.{name}.local` | 4317 (gRPC), 4318 (HTTP) | + +### IAM requirements + +The **task execution role** (`task_execution_role_arn`) needs: +- `ecr:GetAuthorizationToken`, `ecr:BatchGetImage`, `ecr:GetDownloadUrlForLayer` (pull images) +- `logs:CreateLogStream`, `logs:PutLogEvents` (CloudWatch logs) + +The **task role** (`task_role_arn`) needs: +- `s3:GetObject` on the config bucket (config-fetcher sidecar reads configs from S3) + +### Security groups + +ECS services use `awsvpc` networking. Security groups must allow: + +| From | To | Port | Purpose | +| --- | --- | --- | --- | +| Ingester | ClickHouse | 9000 | Telemetry writes | +| SigNoz | ClickHouse | 9000 | Query reads | +| SigNoz | PostgreSQL | 5432 | Metadata | +| SigNoz | Ingester | 4320 | OpAMP management | +| ClickHouse | ClickHouse Keeper | 9181 | Coordination | +| External | SigNoz | 8080 | UI/API access (via ALB) | +| External | Ingester | 4317, 4318 | Telemetry ingestion (via NLB) | diff --git a/docs/examples/ecs/ec2/terraform/casting.yaml b/docs/examples/ecs/ec2/terraform/casting.yaml new file mode 100644 index 0000000..1426460 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/casting.yaml @@ -0,0 +1,9 @@ +apiVersion: v1alpha1 +kind: Installation +metadata: + name: signoz +spec: + deployment: + flavor: terraform + mode: ec2 + platform: ecs diff --git a/docs/examples/ecs/ec2/terraform/casting.yaml.lock b/docs/examples/ecs/ec2/terraform/casting.yaml.lock new file mode 100644 index 0000000..2c2fe18 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/casting.yaml.lock @@ -0,0 +1,647 @@ +apiVersion: v1alpha1 +kind: Installation +metadata: + name: signoz +spec: + deployment: + flavor: terraform + mode: ec2 + platform: ecs + ingester: + spec: + cluster: + replicas: 1 + config: + data: + ingester.yaml: | + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 5s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + metadataexporter: + enabled: true + dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + - metadataexporter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + - metadataexporter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + - metadataexporter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: | + server_endpoint: ws://signoz.signoz.local:4320/v1/opamp + enabled: true + env: + SIGNOZ_OTEL_COLLECTOR_TIMEOUT: 10m + image: signoz/signoz-otel-collector:latest + version: latest + status: + addresses: + otlp: null + config: + data: + ingester.yaml: | + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 5s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + metadataexporter: + enabled: true + dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + - metadataexporter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + - metadataexporter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + - metadataexporter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: | + server_endpoint: ws://signoz.signoz.local:4320/v1/opamp + env: + SIGNOZ_OTEL_COLLECTOR_TIMEOUT: 10m + metastore: + kind: postgres + spec: + cluster: + replicas: 1 + config: {} + enabled: true + env: + POSTGRES_DB: signoz + POSTGRES_PASSWORD: signoz + POSTGRES_USER: signoz + image: postgres:16 + version: "16" + status: + addresses: + dsn: + - tcp://metastore-postgres.signoz.local:5432 + config: {} + env: + POSTGRES_DB: signoz + POSTGRES_PASSWORD: signoz + POSTGRES_USER: signoz + signoz: + spec: + cluster: + replicas: 1 + config: {} + enabled: true + env: + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@metastore-postgres.signoz.local:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://telemetrystore-clickhouse.signoz.local:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + image: signoz/signoz:latest + version: latest + status: + addresses: + apiserver: + - tcp://signoz.signoz.local:8080 + opamp: + - ws://signoz.signoz.local:4320 + config: {} + env: + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@metastore-postgres.signoz.local:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://telemetrystore-clickhouse.signoz.local:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + telemetrykeeper: + kind: clickhousekeeper + spec: + cluster: + replicas: 1 + config: + data: + keeper-0.yaml: | + listen_host: 0.0.0.0 + logger: + level: information + console: true + keeper_server: + four_letter_word_white_list: "*" + coordination_settings: + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: telemetrykeeper-clickhousekeeper.signoz.local + port: 9234 + id: 0 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 + enabled: true + image: clickhouse/clickhouse-keeper:25.5.6 + version: 25.5.6 + status: + addresses: + client: + - tcp://telemetrykeeper-clickhousekeeper.signoz.local:9181 + raft: + - tcp://telemetrykeeper-clickhousekeeper.signoz.local:9234 + config: + data: + keeper-0.yaml: | + listen_host: 0.0.0.0 + logger: + level: information + console: true + keeper_server: + four_letter_word_white_list: "*" + coordination_settings: + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: telemetrykeeper-clickhousekeeper.signoz.local + port: 9234 + id: 0 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 + telemetrystore: + kind: clickhouse + spec: + cluster: + replicas: 0 + shards: 1 + config: + data: + config-0-0.yaml: | + path: /var/lib/clickhouse/ + tmp_path: /var/lib/clickhouse/tmp/ + user_files_path: /var/lib/clickhouse/user_files/ + format_schema_path: /var/lib/clickhouse/format_schemas/ + dictionaries_config: '*_dictionary.xml' + display_name: cluster + distributed_ddl: + path: /clickhouse/task_queue/ddl + http_port: 8123 + interserver_http_port: 9009 + listen_host: 0.0.0.0 + logger: + console: 1 + count: 10 + formatting: + type: console + level: information + size: 1000M + macros: + replica: "00" + shard: "00" + profiles: + default: + allow_simdjson: 0 + load_balancing: random + log_queries: 1 + quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 + user_directories: + users_xml: + path: users.xml + remote_servers: + cluster: + shard: + - replica: + - host: telemetrystore-clickhouse.signoz.local + port: 9000 + zookeeper: + node: + - host: telemetrykeeper-clickhousekeeper.signoz.local + port: 9181 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_thread_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_views_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + part_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + metric_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + asynchronous_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + trace_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + error_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + latency_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + session_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + text_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + zookeeper_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + tcp_port: 9000 + user_defined_executable_functions_config: '*function.yaml' + user_scripts_path: /var/lib/clickhouse/user_scripts/ + users: + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 + functions.yaml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable + enabled: true + image: clickhouse/clickhouse-server:25.5.6 + version: 25.5.6 + status: + addresses: + tcp: + - tcp://telemetrystore-clickhouse.signoz.local:9000 + config: + data: + config-0-0.yaml: | + path: /var/lib/clickhouse/ + tmp_path: /var/lib/clickhouse/tmp/ + user_files_path: /var/lib/clickhouse/user_files/ + format_schema_path: /var/lib/clickhouse/format_schemas/ + dictionaries_config: '*_dictionary.xml' + display_name: cluster + distributed_ddl: + path: /clickhouse/task_queue/ddl + http_port: 8123 + interserver_http_port: 9009 + listen_host: 0.0.0.0 + logger: + console: 1 + count: 10 + formatting: + type: console + level: information + size: 1000M + macros: + replica: "00" + shard: "00" + profiles: + default: + allow_simdjson: 0 + load_balancing: random + log_queries: 1 + quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 + user_directories: + users_xml: + path: users.xml + remote_servers: + cluster: + shard: + - replica: + - host: telemetrystore-clickhouse.signoz.local + port: 9000 + zookeeper: + node: + - host: telemetrykeeper-clickhousekeeper.signoz.local + port: 9181 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_thread_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_views_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + part_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + metric_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + asynchronous_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + trace_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + error_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + latency_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + session_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + text_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + zookeeper_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + tcp_port: 9000 + user_defined_executable_functions_config: '*function.yaml' + user_scripts_path: /var/lib/clickhouse/user_scripts/ + users: + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 + functions.yaml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/main.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/main.tf.json new file mode 100644 index 0000000..7cdd6a4 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/main.tf.json @@ -0,0 +1,30 @@ +{ + "terraform": { + "required_version": ">= 1.0", + "required_providers": { + "aws": { + "source": "hashicorp/aws", + "version": ">= 5.0" + } + } + }, + "provider": { + "aws": { + "region": "${var.region}" + } + }, + "module": { + "signoz": { + "source": "./module", + "region": "${var.region}", + "ecs_cluster_id": "${var.ecs_cluster_id}", + "subnet_ids": "${var.subnet_ids}", + "security_group_ids": "${var.security_group_ids}", + "vpc_id": "${var.vpc_id}", + "config_bucket": "${var.config_bucket}", + "task_role_arn": "${var.task_role_arn}", + "task_execution_role_arn": "${var.task_execution_role_arn}", + "capacity_provider": "${var.capacity_provider}" + } + } +} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester.tf.json new file mode 100644 index 0000000..40e043d --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester.tf.json @@ -0,0 +1,125 @@ +{ + "locals": { + "containers": [ + { + "name": "config-fetcher", + "image": "amazon/aws-cli:2.27.32", + "essential": false, + "entryPoint": ["/bin/sh", "-c"], + "command": ["aws s3 cp s3://${var.config_bucket}/signoz/ingester/ /configs/ingester/ --recursive"], + "mountPoints": [ + { + "sourceVolume": "ingester-config", + "containerPath": "/configs/ingester" + } + ], + "cpu": 10, + "memory": 102, + "memoryReservation": 102 + }, + { + "name": "ingester", + "image": "signoz/signoz-otel-collector:latest", + "essential": true, + "entryPoint": ["/bin/sh", "-c"], + "command": ["/signoz-otel-collector migrate sync check && /signoz-otel-collector --config=/conf/ingester.yaml --manager-config=/conf/opamp.yaml --copy-path=/var/tmp/collector-config.yaml"], + "environment": [{"name":"SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN","value":"tcp://telemetrystore-clickhouse.signoz.local:9000"},{"name":"SIGNOZ_OTEL_COLLECTOR_TIMEOUT","value":"10m"}], + "portMappings": [ + {"name": "grpc", "containerPort": 4317, "protocol": "tcp", "appProtocol": "grpc"}, + {"name": "http", "containerPort": 4318, "protocol": "tcp", "appProtocol": "http"} + ], + "mountPoints": [ + { + "sourceVolume": "ingester-config", + "containerPath": "/conf" + } + ], + "dependsOn": [ + {"containerName": "config-fetcher", "condition": "SUCCESS"} + ], + "cpu": 512, + "memory": 512, + "memoryReservation": 512, + "healthCheck": { + "command": ["CMD-SHELL", "wget --spider -q localhost:13133 || exit 1"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + } + } + ] + }, + "resource": { + "aws_s3_object": { + "ingester_configs": { + "for_each": "${fileset(\"${path.module}/ingester\", \"*.yaml\")}", + "bucket": "${var.config_bucket}", + "key": "signoz/ingester/${each.value}", + "source": "${path.module}/ingester/${each.value}", + "etag": "${filemd5(\"${path.module}/ingester/${each.value}\")}" + } + }, + "aws_ecs_task_definition": { + "ingester": { + "family": "signoz-ingester", + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${var.task_role_arn}", + "execution_role_arn": "${var.task_execution_role_arn}", + "container_definitions": "${jsonencode(local.containers)}", + "volume": [ + { + "name": "ingester-config", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + } + ], + "depends_on": ["aws_s3_object.ingester_configs"] + } + }, + "aws_service_discovery_service": { + "ingester": { + "name": "ingester", + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + }, + "aws_ecs_service": { + "ingester": { + "name": "signoz-ingester", + "cluster": "${var.ecs_cluster_id}", + "task_definition": "${aws_ecs_task_definition.ingester.arn}", + "desired_count": 1, + "capacity_provider_strategy": [ + { + "capacity_provider": "${var.capacity_provider}", + "weight": 1, + "base": 0 + } + ], + "network_configuration": { + "subnets": "${var.subnet_ids}", + "security_groups": "${var.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.ingester.arn}" + }, +"depends_on": [ + "aws_ecs_service.signoz", + "aws_ecs_service.telemetrystore" + ] + } + } + } +} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester/ingester.yaml b/docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester/ingester.yaml new file mode 100644 index 0000000..0e5b7c8 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester/ingester.yaml @@ -0,0 +1,132 @@ +connectors: + signozmeter: + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + metrics_flush_interval: 1h +exporters: + clickhouselogsexporter: + dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_logs + sending_queue: + enabled: false + timeout: 45s + use_new_schema: true + clickhousetraces: + datasource: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + sending_queue: + enabled: false + timeout: 45s + use_new_schema: true + metadataexporter: + cache: + provider: in_memory + dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_metadata + enabled: true + timeout: 45s + signozclickhousemeter: + dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_meter + sending_queue: + enabled: false + timeout: 45s + signozclickhousemetrics: + dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_metrics + sending_queue: + enabled: false + timeout: 45s +extensions: + pprof: + endpoint: 0.0.0.0:1777 + signoz_health_check: + endpoint: 0.0.0.0:13133 +processors: + batch: + send_batch_max_size: 55000 + send_batch_size: 50000 + timeout: 5s + batch/meter: + send_batch_max_size: 25000 + send_batch_size: 20000 + timeout: 5s + signozspanmetrics/delta: + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + dimensions: + - default: default + name: service.namespace + - default: default + name: deployment.environment + - name: signoz.collector.id + - name: service.version + dimensions_cache_size: 100000 + enable_exp_histogram: true + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +service: + extensions: + - signoz_health_check + - pprof + pipelines: + logs: + exporters: + - clickhouselogsexporter + - signozmeter + - metadataexporter + processors: + - batch + receivers: + - otlp + metrics: + exporters: + - signozclickhousemetrics + - signozmeter + - metadataexporter + processors: + - batch + receivers: + - otlp + metrics/meter: + exporters: + - signozclickhousemeter + processors: + - batch/meter + receivers: + - signozmeter + traces: + exporters: + - clickhousetraces + - signozmeter + - metadataexporter + processors: + - signozspanmetrics/delta + - batch + receivers: + - otlp + telemetry: + logs: + encoding: json diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester/opamp.yaml b/docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester/opamp.yaml new file mode 100644 index 0000000..46c46c2 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester/opamp.yaml @@ -0,0 +1 @@ +server_endpoint: ws://signoz.signoz.local:4320/v1/opamp diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/main.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/module/main.tf.json new file mode 100644 index 0000000..9868aa9 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/module/main.tf.json @@ -0,0 +1,10 @@ +{ + "resource": { + "aws_service_discovery_private_dns_namespace": { + "main": { + "name": "signoz.local", + "vpc": "${var.vpc_id}" + } + } + } +} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/metastore.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/module/metastore.tf.json new file mode 100644 index 0000000..df1ccf2 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/module/metastore.tf.json @@ -0,0 +1,89 @@ +{ + "locals": { + "containers": [ + { + "name": "signoz-metastore-postgres-0", + "image": "postgres:16", + "essential": true, + "environment": [{"name":"POSTGRES_DB","value":"signoz"},{"name":"POSTGRES_PASSWORD","value":"signoz"},{"name":"POSTGRES_USER","value":"signoz"}], + "portMappings": [ + {"name": "postgres", "containerPort": 5432, "protocol": "tcp"} + ], + "mountPoints": [ + { + "sourceVolume": "postgres-data", + "containerPath": "/var/lib/postgresql/data" + } + ], + "cpu": 256, + "memory": 256, + "memoryReservation": 256, + "healthCheck": { + "command": ["CMD-SHELL", "pg_isready -U postgres || exit 1"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + } + } + ] + }, + "resource": { + "aws_ecs_task_definition": { + "metastore": { + "family": "signoz-metastore-postgres-0", + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${var.task_role_arn}", + "execution_role_arn": "${var.task_execution_role_arn}", + "container_definitions": "${jsonencode(local.containers)}", + "volume": [ + { + "name": "postgres-data", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + } + ] + } + }, + "aws_service_discovery_service": { + "metastore": { + "name": "metastore-postgres", + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + }, + "aws_ecs_service": { + "metastore": { + "name": "signoz-metastore-postgres-0", + "cluster": "${var.ecs_cluster_id}", + "task_definition": "${aws_ecs_task_definition.metastore.arn}", + "desired_count": 1, + "capacity_provider_strategy": [ + { + "capacity_provider": "${var.capacity_provider}", + "weight": 1, + "base": 0 + } + ], + "network_configuration": { + "subnets": "${var.subnet_ids}", + "security_groups": "${var.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.metastore.arn}" + } + } + } + } +} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/outputs.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/module/outputs.tf.json new file mode 100644 index 0000000..d2799c0 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/module/outputs.tf.json @@ -0,0 +1,52 @@ +{ + "output": { + "ecs_cluster_id": { + "description": "ECS cluster ID", + "value": "${var.ecs_cluster_id}" + }, + "namespace_id": { + "description": "Cloud Map private DNS namespace ID", + "value": "${aws_service_discovery_private_dns_namespace.main.id}" + }, + "namespace_name": { + "description": "Cloud Map private DNS namespace name", + "value": "${aws_service_discovery_private_dns_namespace.main.name}" + }, + "subnet_ids": { + "description": "Subnet IDs used by ECS services", + "value": "${var.subnet_ids}" + }, + "security_group_ids": { + "description": "Security group IDs used by ECS services", + "value": "${var.security_group_ids}" + }, + "signoz_service_arn": { + "description": "SigNoz ECS service ARN (target for ALB on port 8080)", + "value": "${aws_ecs_service.signoz.id}" + }, + "signoz_service_name": { + "description": "SigNoz ECS service name", + "value": "${aws_ecs_service.signoz.name}" + }, + "ingester_service_arn": { + "description": "Ingester ECS service ARN (target for NLB on port 4317/4318)", + "value": "${aws_ecs_service.ingester.id}" + }, + "ingester_service_name": { + "description": "Ingester ECS service name", + "value": "${aws_ecs_service.ingester.name}" + }, + "telemetrystore_service_name": { + "description": "TelemetryStore ECS service name", + "value": "${aws_ecs_service.telemetrystore.name}" + }, + "telemetrykeeper_service_name": { + "description": "TelemetryKeeper ECS service name", + "value": "${aws_ecs_service.telemetrykeeper.name}" + }, + "metastore_service_name": { + "description": "MetaStore ECS service name", + "value": "${aws_ecs_service.metastore.name}" + } + } +} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/signoz.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/module/signoz.tf.json new file mode 100644 index 0000000..e3f57a5 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/module/signoz.tf.json @@ -0,0 +1,79 @@ +{ + "locals": { + "containers": [ + { + "name": "signoz", + "image": "signoz/signoz:latest", + "essential": true, + "environment": [{"name":"SIGNOZ_SQLSTORE_POSTGRES_DSN","value":"postgres://signoz:signoz@metastore-postgres.signoz.local:5432/signoz?sslmode=disable"},{"name":"SIGNOZ_SQLSTORE_PROVIDER","value":"postgres"},{"name":"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN","value":"tcp://telemetrystore-clickhouse.signoz.local:9000"},{"name":"SIGNOZ_TELEMETRYSTORE_PROVIDER","value":"clickhouse"}], + "portMappings": [ + {"name": "http", "containerPort": 8080, "protocol": "tcp", "appProtocol": "http"}, + {"name": "opamp", "containerPort": 4320, "protocol": "tcp"} + ], + "cpu": 512, + "memory": 512, + "memoryReservation": 512, + "healthCheck": { + "command": ["CMD-SHELL", "wget --spider -q localhost:8080/api/v1/health || exit 1"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + } + } + ] + }, + "resource": { + "aws_ecs_task_definition": { + "signoz": { + "family": "signoz-signoz", + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${var.task_role_arn}", + "execution_role_arn": "${var.task_execution_role_arn}", + "container_definitions": "${jsonencode(local.containers)}" + } + }, + "aws_service_discovery_service": { + "signoz": { + "name": "signoz", + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + }, + "aws_ecs_service": { + "signoz": { + "name": "signoz-signoz", + "cluster": "${var.ecs_cluster_id}", + "task_definition": "${aws_ecs_task_definition.signoz.arn}", + "desired_count": 1, + "capacity_provider_strategy": [ + { + "capacity_provider": "${var.capacity_provider}", + "weight": 1, + "base": 0 + } + ], + "network_configuration": { + "subnets": "${var.subnet_ids}", + "security_groups": "${var.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.signoz.arn}" + }, +"depends_on": [ + "aws_ecs_service.metastore", + "aws_ecs_service.telemetrystore" + ] + } + } + } +} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrykeeper.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrykeeper.tf.json new file mode 100644 index 0000000..68c126d --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrykeeper.tf.json @@ -0,0 +1,130 @@ +{ + "locals": { + "containers": [ + { + "name": "config-fetcher", + "image": "amazon/aws-cli:2.27.32", + "essential": false, + "entryPoint": ["/bin/sh", "-c"], + "command": ["aws s3 cp s3://${var.config_bucket}/signoz/telemetrykeeper/clickhousekeeper/ /configs/telemetrykeeper/ --recursive"], + "mountPoints": [ + { + "sourceVolume": "keeper-config", + "containerPath": "/configs/telemetrykeeper" + } + ], + "cpu": 10, + "memory": 102, + "memoryReservation": 102 + }, + { + "name": "signoz-telemetrykeeper-clickhousekeeper-0", + "image": "clickhouse/clickhouse-keeper:25.5.6", + "essential": true, + "entryPoint": ["/usr/bin/clickhouse-keeper", "--config-file=/etc/clickhouse-keeper/keeper.yaml"], + "portMappings": [ + {"name": "client", "containerPort": 9181, "protocol": "tcp"}, + {"name": "raft", "containerPort": 9234, "protocol": "tcp"} + ], + "mountPoints": [ + { + "sourceVolume": "keeper-data", + "containerPath": "/var/lib/clickhouse-keeper" + }, + { + "sourceVolume": "keeper-config", + "containerPath": "/etc/clickhouse-keeper" + } + ], + "dependsOn": [ + {"containerName": "config-fetcher", "condition": "SUCCESS"} + ], + "cpu": 256, + "memory": 512, + "memoryReservation": 512, + "healthCheck": { + "command": ["CMD-SHELL", "echo ruok | nc localhost 9181 || exit 1"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + } + } + ] + }, + "resource": { + "aws_s3_object": { + "telemetrykeeper_configs": { + "for_each": "${fileset(\"${path.module}/telemetrykeeper/clickhousekeeper\", \"**\")}", + "bucket": "${var.config_bucket}", + "key": "signoz/telemetrykeeper/clickhousekeeper/${each.value}", + "source": "${path.module}/telemetrykeeper/clickhousekeeper/${each.value}", + "etag": "${filemd5(\"${path.module}/telemetrykeeper/clickhousekeeper/${each.value}\")}" + } + }, + "aws_ecs_task_definition": { + "telemetrykeeper": { + "family": "signoz-telemetrykeeper-clickhousekeeper-0", + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${var.task_role_arn}", + "execution_role_arn": "${var.task_execution_role_arn}", + "container_definitions": "${jsonencode(local.containers)}", + "volume": [ + { + "name": "keeper-config", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + }, + { + "name": "keeper-data", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + } + ], + "depends_on": ["aws_s3_object.telemetrykeeper_configs"] + } + }, + "aws_service_discovery_service": { + "telemetrykeeper": { + "name": "telemetrykeeper-clickhousekeeper", + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + }, + "aws_ecs_service": { + "telemetrykeeper": { + "name": "signoz-telemetrykeeper-clickhousekeeper-0", + "cluster": "${var.ecs_cluster_id}", + "task_definition": "${aws_ecs_task_definition.telemetrykeeper.arn}", + "desired_count": 1, + "capacity_provider_strategy": [ + { + "capacity_provider": "${var.capacity_provider}", + "weight": 1, + "base": 0 + } + ], + "network_configuration": { + "subnets": "${var.subnet_ids}", + "security_groups": "${var.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.telemetrykeeper.arn}" + } + } + } + } +} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrykeeper/clickhousekeeper/keeper-0.yaml b/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrykeeper/clickhousekeeper/keeper-0.yaml new file mode 100644 index 0000000..89dd46c --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrykeeper/clickhousekeeper/keeper-0.yaml @@ -0,0 +1,22 @@ +keeper_server: + coordination_settings: + force_sync: false + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + snapshot_distance: 100000 + snapshots_to_keep: 3 + four_letter_word_white_list: '*' + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: telemetrykeeper-clickhousekeeper.signoz.local + id: 0 + port: 9234 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 +listen_host: 0.0.0.0 +logger: + console: true + level: information diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore.tf.json new file mode 100644 index 0000000..4afa75e --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore.tf.json @@ -0,0 +1,161 @@ +{ + "locals": { + "containers": [ + { + "name": "init-clickhouse", + "image": "alpine:3.18.2", + "essential": false, + "command": ["/bin/sh", "-c", "cd /tmp && wget -O histogram-quantile.tar.gz 'https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2Fv0.0.1/histogram-quantile_linux_amd64.tar.gz' && tar -xzf histogram-quantile.tar.gz && mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile"], + "mountPoints": [ + { + "sourceVolume": "shared-binary-volume", + "containerPath": "/var/lib/clickhouse/user_scripts" + } + ], + "cpu": 256, + "memory": 256, + "memoryReservation": 256 + }, + { + "name": "config-fetcher", + "image": "amazon/aws-cli:2.27.32", + "essential": false, + "entryPoint": ["/bin/sh", "-c"], + "command": ["aws s3 cp s3://${var.config_bucket}/signoz/telemetrystore/clickhouse/ /configs/telemetrystore/ --recursive"], + "mountPoints": [ + { + "sourceVolume": "telemetrystore-clickhouse-config", + "containerPath": "/configs/telemetrystore" + } + ], + "cpu": 10, + "memory": 102, + "memoryReservation": 102 + }, + { + "name": "signoz-telemetrystore-clickhouse-0-0", + "image": "clickhouse/clickhouse-server:25.5.6", + "essential": true, + "environment": [{"name":"CLICKHOUSE_SKIP_USER_SETUP","value":"1"}], + "portMappings": [ + {"name": "native", "containerPort": 9000, "protocol": "tcp"}, + {"name": "http", "containerPort": 8123, "protocol": "tcp", "appProtocol": "http"}, + {"name": "prometheus", "containerPort": 9363, "protocol": "tcp", "appProtocol": "http"} + ], + "mountPoints": [ + { + "sourceVolume": "shared-binary-volume", + "containerPath": "/var/lib/clickhouse/user_scripts" + }, + { + "sourceVolume": "telemetrystore-clickhouse-config", + "containerPath": "/etc/clickhouse-server/config.d" + }, + { + "sourceVolume": "clickhouse-data", + "containerPath": "/var/lib/clickhouse" + } + ], + "dependsOn": [ + {"containerName": "init-clickhouse", "condition": "SUCCESS"}, + {"containerName": "config-fetcher", "condition": "SUCCESS"} + ], + "cpu": 1024, + "memory": 512, + "memoryReservation": 512, + "healthCheck": { + "command": ["CMD-SHELL", "wget --spider -q 0.0.0.0:8123/ping || exit 1"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + } + } + ] + }, + "resource": { + "aws_s3_object": { + "telemetrystore_configs": { + "for_each": "${fileset(\"${path.module}/telemetrystore/clickhouse\", \"**\")}", + "bucket": "${var.config_bucket}", + "key": "signoz/telemetrystore/clickhouse/${each.value}", + "source": "${path.module}/telemetrystore/clickhouse/${each.value}", + "etag": "${filemd5(\"${path.module}/telemetrystore/clickhouse/${each.value}\")}" + } + }, + "aws_ecs_task_definition": { + "telemetrystore": { + "family": "signoz-telemetrystore-clickhouse-0-0", + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${var.task_role_arn}", + "execution_role_arn": "${var.task_execution_role_arn}", + "container_definitions": "${jsonencode(local.containers)}", + "volume": [ + { + "name": "shared-binary-volume", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + }, + { + "name": "telemetrystore-clickhouse-config", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + }, + { + "name": "clickhouse-data", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + } + ], + "depends_on": ["aws_s3_object.telemetrystore_configs"] + } + }, + "aws_service_discovery_service": { + "telemetrystore": { + "name": "telemetrystore-clickhouse", + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + }, + "aws_ecs_service": { + "telemetrystore": { + "name": "signoz-telemetrystore-clickhouse-0-0", + "cluster": "${var.ecs_cluster_id}", + "task_definition": "${aws_ecs_task_definition.telemetrystore.arn}", + "desired_count": 1, + "capacity_provider_strategy": [ + { + "capacity_provider": "${var.capacity_provider}", + "weight": 1, + "base": 0 + } + ], + "network_configuration": { + "subnets": "${var.subnet_ids}", + "security_groups": "${var.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.telemetrystore.arn}" + }, +"depends_on": [ + "aws_ecs_service.telemetrykeeper" + ] + } + } + } +} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore/clickhouse/config-0-0.yaml b/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore/clickhouse/config-0-0.yaml new file mode 100644 index 0000000..ff2ff57 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore/clickhouse/config-0-0.yaml @@ -0,0 +1,96 @@ +asynchronous_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE +dictionaries_config: '*_dictionary.xml' +display_name: cluster +distributed_ddl: + path: /clickhouse/task_queue/ddl +error_log: + ttl: event_date + INTERVAL 1 DAY DELETE +format_schema_path: /var/lib/clickhouse/format_schemas/ +http_port: 8123 +interserver_http_port: 9009 +latency_log: + ttl: event_date + INTERVAL 1 DAY DELETE +listen_host: 0.0.0.0 +logger: + console: 1 + count: 10 + formatting: + type: console + level: information + size: 1000M +macros: + replica: "00" + shard: "00" +metric_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE +part_log: + ttl: event_date + INTERVAL 1 DAY DELETE +path: /var/lib/clickhouse/ +processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE +profiles: + default: + allow_simdjson: 0 + load_balancing: random + log_queries: 1 +query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: event_date + INTERVAL 1 DAY DELETE +query_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE +query_thread_log: + ttl: event_date + INTERVAL 1 DAY DELETE +query_views_log: + ttl: event_date + INTERVAL 1 DAY DELETE +quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 +remote_servers: + cluster: + shard: + - replica: + - host: telemetrystore-clickhouse.signoz.local + port: 9000 +session_log: + ttl: event_date + INTERVAL 1 DAY DELETE +tcp_port: 9000 +text_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE +tmp_path: /var/lib/clickhouse/tmp/ +trace_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE +user_defined_executable_functions_config: '*function.yaml' +user_directories: + users_xml: + path: users.xml +user_files_path: /var/lib/clickhouse/user_files/ +user_scripts_path: /var/lib/clickhouse/user_scripts/ +users: + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 +zookeeper: + node: + - host: telemetrykeeper-clickhousekeeper.signoz.local + port: 9181 +zookeeper_log: + ttl: event_date + INTERVAL 1 DAY DELETE diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore/clickhouse/functions.yaml b/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore/clickhouse/functions.yaml new file mode 100644 index 0000000..9a88469 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore/clickhouse/functions.yaml @@ -0,0 +1,13 @@ +functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore_migrator.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore_migrator.tf.json new file mode 100644 index 0000000..9019493 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore_migrator.tf.json @@ -0,0 +1,48 @@ +{ + "locals": { + "containers": [ + { + "name": "telemetrystore-migrator", + "image": "signoz/signoz-otel-collector:latest", + "essential": true, + "entryPoint": ["/bin/sh", "-c"], + "command": ["/signoz-otel-collector migrate ready && /signoz-otel-collector migrate bootstrap && /signoz-otel-collector migrate sync up && /signoz-otel-collector migrate async up"], + "environment": [{"name":"SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN","value":"tcp://telemetrystore-clickhouse.signoz.local:9000"},{"name":"SIGNOZ_OTEL_COLLECTOR_TIMEOUT","value":"10m"}], + "cpu": 256, + "memory": 512 + } + ] + }, + "resource": { + "aws_ecs_task_definition": { + "telemetrystore_migrator": { + "family": "signoz-telemetrystore-migrator", + "network_mode": "awsvpc", + "requires_compatibilities": ["FARGATE"], + "cpu": 256, + "memory": 512, + "task_role_arn": "${var.task_role_arn}", + "execution_role_arn": "${var.task_execution_role_arn}", + "container_definitions": "${jsonencode(local.containers)}" + } + } + }, + "data": { + "aws_ecs_task_execution": { + "telemetrystore_migrator": { + "cluster": "${var.ecs_cluster_id}", + "task_definition": "${aws_ecs_task_definition.telemetrystore_migrator.arn}", + "desired_count": 1, + "launch_type": "FARGATE", + "network_configuration": { + "subnets": "${var.subnet_ids}", + "security_groups": "${var.security_group_ids}", + "assign_public_ip": false + }, + "depends_on": [ + "aws_ecs_service.telemetrystore" + ] + } + } + } +} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/variables.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/module/variables.tf.json new file mode 100644 index 0000000..fd7e081 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/module/variables.tf.json @@ -0,0 +1,40 @@ +{ + "variable": { + "region": { + "description": "AWS region", + "type": "string" + }, + "ecs_cluster_id": { + "description": "ID of the existing ECS cluster to deploy services into", + "type": "string" + }, + "subnet_ids": { + "description": "List of subnet IDs for ECS service networking (awsvpc)", + "type": "list(string)" + }, + "security_group_ids": { + "description": "List of security group IDs for ECS service networking (awsvpc)", + "type": "list(string)" + }, + "vpc_id": { + "description": "VPC ID for the private DNS namespace", + "type": "string" + }, + "config_bucket": { + "description": "S3 bucket name for storing config files", + "type": "string" + }, + "task_role_arn": { + "description": "IAM role ARN for ECS tasks", + "type": "string" + }, + "task_execution_role_arn": { + "description": "IAM role ARN for ECS task execution (pull images, write logs)", + "type": "string" + }, + "capacity_provider": { + "description": "Name of the ECS capacity provider", + "type": "string" + } + } +} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/terraform.tfvars.json b/docs/examples/ecs/ec2/terraform/pours/deployment/terraform.tfvars.json new file mode 100644 index 0000000..cceaa20 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/terraform.tfvars.json @@ -0,0 +1,11 @@ +{ + "region": "", + "ecs_cluster_id": "", + "subnet_ids": [""], + "security_group_ids": [""], + "vpc_id": "", + "config_bucket": "", + "task_role_arn": "", + "task_execution_role_arn": "", + "capacity_provider": "" +} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/variables.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/variables.tf.json new file mode 100644 index 0000000..fd7e081 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/variables.tf.json @@ -0,0 +1,40 @@ +{ + "variable": { + "region": { + "description": "AWS region", + "type": "string" + }, + "ecs_cluster_id": { + "description": "ID of the existing ECS cluster to deploy services into", + "type": "string" + }, + "subnet_ids": { + "description": "List of subnet IDs for ECS service networking (awsvpc)", + "type": "list(string)" + }, + "security_group_ids": { + "description": "List of security group IDs for ECS service networking (awsvpc)", + "type": "list(string)" + }, + "vpc_id": { + "description": "VPC ID for the private DNS namespace", + "type": "string" + }, + "config_bucket": { + "description": "S3 bucket name for storing config files", + "type": "string" + }, + "task_role_arn": { + "description": "IAM role ARN for ECS tasks", + "type": "string" + }, + "task_execution_role_arn": { + "description": "IAM role ARN for ECS task execution (pull images, write logs)", + "type": "string" + }, + "capacity_provider": { + "description": "Name of the ECS capacity provider", + "type": "string" + } + } +} diff --git a/docs/examples/kubernetes/README.md b/docs/examples/kubernetes/README.md new file mode 100644 index 0000000..34309f7 --- /dev/null +++ b/docs/examples/kubernetes/README.md @@ -0,0 +1,6 @@ +# Kubernetes + +| Flavor | Description | +| --- | --- | +| [kustomize](kustomize/) | Kubernetes manifests with Kustomize overlays | +| [helm](helm/) | Helm chart values file | diff --git a/docs/examples/kubernetes/helm-patches/README.md b/docs/examples/kubernetes/helm-patches/README.md new file mode 100644 index 0000000..84e2e06 --- /dev/null +++ b/docs/examples/kubernetes/helm-patches/README.md @@ -0,0 +1,129 @@ +# Helm with Patches + +| Field | Value | +| --- | --- | +| **Mode** | `kubernetes` | +| **Flavor** | `helm` | +| **Platform** | `-` | + +## Overview + +Extends the base [kubernetes/helm](../helm/) example with production patches for AWS. Demonstrates setting resource limits, dedicated node scheduling, and persistence configuration on the generated `values.yaml` using `spec.patches`. + +For patch reference, see [Patches](../../../concepts/patches.md). + +## Prerequisites + +- Kubernetes cluster (1.24+) +- Helm 3.x +- SigNoz Helm chart repo (`https://charts.signoz.io`) + +## Configuration + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + flavor: helm + mode: kubernetes + patches: + # Resource limits on ClickHouse + - target: "deployment/values.yaml" + operations: + - op: add + path: /clickhouse/resources + value: + requests: + cpu: "2" + memory: "4Gi" + limits: + cpu: "4" + memory: "8Gi" + + # Resource limits on SigNoz + - target: "deployment/values.yaml" + operations: + - op: add + path: /signoz/resources + value: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "1" + memory: "2Gi" + + # Tolerations and nodeSelector for dedicated nodes + - target: "deployment/values.yaml" + operations: + - op: add + path: /signoz/tolerations + value: + - key: "dedicated" + operator: "Equal" + value: "signoz" + effect: "NoSchedule" + - op: add + path: /signoz/nodeSelector + value: + node-role.kubernetes.io/observability: "" + + # Resource limits on OTel Collector (ingester) + - target: "deployment/values.yaml" + operations: + - op: add + path: /otelCollector/resources + value: + requests: + cpu: "1" + memory: "2Gi" + limits: + cpu: "2" + memory: "4Gi" + + # ClickHouse persistence + - target: "deployment/values.yaml" + operations: + - op: add + path: /clickhouse/persistence + value: + enabled: true + storageClass: gp3 + size: 100Gi +``` + +## Deploy + +```bash +foundryctl cast -f casting.yaml +``` + +Or step by step: + +```bash +foundryctl forge -f casting.yaml +helm install signoz signoz/signoz -f pours/deployment/values.yaml -n signoz --create-namespace +``` + +## Generated output + +```text +pours/deployment/ + values.yaml +``` + +## Customization + +### Resource limits + +Each component maps to a top-level key in the Helm chart values: `clickhouse`, `signoz`, `otelCollector`. Set `resources.requests` and `resources.limits` on each. + +### Tolerations and nodeSelector + +Schedules SigNoz on nodes labeled `node-role.kubernetes.io/observability` with a `dedicated=signoz:NoSchedule` toleration. + +### Persistence + +Configures ClickHouse persistent storage with a cloud provider storage class. Adjust `storageClass` for your provider: `gp3` on AWS, `pd-ssd` on GCP, `managed-premium` on Azure. diff --git a/docs/examples/kubernetes/helm-patches/casting.yaml b/docs/examples/kubernetes/helm-patches/casting.yaml new file mode 100644 index 0000000..a6b8695 --- /dev/null +++ b/docs/examples/kubernetes/helm-patches/casting.yaml @@ -0,0 +1,71 @@ +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + flavor: helm + mode: kubernetes + patches: + # 1. Set resource requests and limits on ClickHouse + - target: "deployment/values.yaml" + operations: + - op: add + path: /clickhouse/resources + value: + requests: + cpu: "2" + memory: "4Gi" + limits: + cpu: "4" + memory: "8Gi" + + # 2. Set resource requests and limits on SigNoz + - target: "deployment/values.yaml" + operations: + - op: add + path: /signoz/resources + value: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "1" + memory: "2Gi" + + # 3. Add tolerations and nodeSelector for dedicated observability nodes + - target: "deployment/values.yaml" + operations: + - op: add + path: /signoz/tolerations + value: + - key: "dedicated" + operator: "Equal" + value: "signoz" + effect: "NoSchedule" + - op: add + path: /signoz/nodeSelector + value: + node-role.kubernetes.io/observability: "" + + # 4. Set resource requests and limits on OTel Collector (ingester) + - target: "deployment/values.yaml" + operations: + - op: add + path: /otelCollector/resources + value: + requests: + cpu: "1" + memory: "2Gi" + limits: + cpu: "2" + memory: "4Gi" + + # 5. Set persistence on ClickHouse + - target: "deployment/values.yaml" + operations: + - op: add + path: /clickhouse/persistence + value: + enabled: true + storageClass: gp3 + size: 100Gi diff --git a/docs/examples/kubernetes/helm-patches/casting.yaml.lock b/docs/examples/kubernetes/helm-patches/casting.yaml.lock new file mode 100644 index 0000000..476dca2 --- /dev/null +++ b/docs/examples/kubernetes/helm-patches/casting.yaml.lock @@ -0,0 +1,603 @@ +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + flavor: helm + mode: kubernetes + ingester: + spec: + cluster: + replicas: 1 + config: + data: + ingester.yaml: | + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 1s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 1s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://signoz-telemetrystore-clickhouse:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://signoz-telemetrystore-clickhouse:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://signoz-telemetrystore-clickhouse:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://signoz-telemetrystore-clickhouse:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: | + server_endpoint: tcp://signoz:4320/v1/opamp + enabled: true + image: signoz/signoz-otel-collector:latest + version: latest + status: + addresses: + otlp: null + config: + data: + ingester.yaml: | + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 1s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 1s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://signoz-telemetrystore-clickhouse:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://signoz-telemetrystore-clickhouse:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://signoz-telemetrystore-clickhouse:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://signoz-telemetrystore-clickhouse:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: | + server_endpoint: tcp://signoz:4320/v1/opamp + metastore: + kind: postgres + spec: + cluster: + replicas: 1 + config: {} + enabled: true + env: + POSTGRES_DB: signoz + POSTGRES_PASSWORD: signoz + POSTGRES_USER: signoz + image: postgres:16 + version: "16" + status: + addresses: + dsn: + - postgres://signoz-metastore-postgres:5432 + config: {} + env: + POSTGRES_DB: signoz + POSTGRES_PASSWORD: signoz + POSTGRES_USER: signoz + patches: + - operations: + - op: add + path: /clickhouse/resources + value: + limits: + cpu: "4" + memory: 8Gi + requests: + cpu: "2" + memory: 4Gi + target: deployment/values.yaml + - operations: + - op: add + path: /signoz/resources + value: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: 500m + memory: 1Gi + target: deployment/values.yaml + - operations: + - op: add + path: /signoz/tolerations + value: + - effect: NoSchedule + key: dedicated + operator: Equal + value: signoz + - op: add + path: /signoz/nodeSelector + value: + node-role.kubernetes.io/observability: "" + target: deployment/values.yaml + - operations: + - op: add + path: /otelCollector/resources + value: + limits: + cpu: "2" + memory: 4Gi + requests: + cpu: "1" + memory: 2Gi + target: deployment/values.yaml + - operations: + - op: add + path: /clickhouse/persistence + value: + enabled: true + size: 100Gi + storageClass: gp3 + target: deployment/values.yaml + signoz: + spec: + cluster: + replicas: 1 + config: {} + enabled: true + env: + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@signoz-metastore-postgres:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://signoz-telemetrystore-clickhouse:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + image: signoz/signoz:latest + version: latest + status: + addresses: + apiserver: null + opamp: + - tcp://signoz:4320 + config: {} + env: + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@signoz-metastore-postgres:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://signoz-telemetrystore-clickhouse:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + telemetrykeeper: + kind: clickhousekeeper + spec: + cluster: + replicas: 1 + config: + data: + keeper-0.yaml: | + listen_host: 0.0.0.0 + logger: + level: information + console: true + keeper_server: + four_letter_word_white_list: "*" + coordination_settings: + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: signoz-telemetrykeeper-zookeeper-0 + port: 9234 + id: 0 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 + enabled: true + image: clickhouse/clickhouse-keeper:25.5.6 + version: 25.5.6 + status: + addresses: + client: + - tcp://signoz-telemetrykeeper-zookeeper-0:9181 + raft: + - tcp://signoz-telemetrykeeper-zookeeper-0:9234 + config: + data: + keeper-0.yaml: | + listen_host: 0.0.0.0 + logger: + level: information + console: true + keeper_server: + four_letter_word_white_list: "*" + coordination_settings: + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: signoz-telemetrykeeper-zookeeper-0 + port: 9234 + id: 0 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 + telemetrystore: + kind: clickhouse + spec: + cluster: + replicas: 0 + shards: 1 + config: + data: + config.yaml: | + path: /var/lib/clickhouse/ + tmp_path: /var/lib/clickhouse/tmp/ + user_files_path: /var/lib/clickhouse/user_files/ + format_schema_path: /var/lib/clickhouse/format_schemas/ + dictionaries_config: '*_dictionary.xml' + display_name: cluster + distributed_ddl: + path: /clickhouse/task_queue/ddl + http_port: 8123 + interserver_http_port: 9009 + listen_host: 0.0.0.0 + logger: + console: 1 + count: 10 + formatting: + type: console + level: information + size: 1000M + macros: + replica: "01" + shard: "01" + profiles: + default: + load_balancing: random + log_queries: 1 + max_memory_usage: 10000000000 + quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 + user_directories: + users_xml: + path: users.xml + remote_servers: + cluster: + shard: + - replica: + host: signoz-telemetrystore-clickhouse + port: 9000 + zookeeper: + node: + - host: signoz-telemetrykeeper-zookeeper-0 + port: 9181 + tcp_port: 9000 + user_defined_executable_functions_config: '*function.yaml' + user_scripts_path: /var/lib/clickhouse/user_scripts/ + users: + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 + functions.yaml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable + enabled: true + image: clickhouse/clickhouse-server:25.5.6 + version: 25.5.6 + status: + addresses: + tcp: + - tcp://signoz-telemetrystore-clickhouse:9000 + config: + data: + config.yaml: | + path: /var/lib/clickhouse/ + tmp_path: /var/lib/clickhouse/tmp/ + user_files_path: /var/lib/clickhouse/user_files/ + format_schema_path: /var/lib/clickhouse/format_schemas/ + dictionaries_config: '*_dictionary.xml' + display_name: cluster + distributed_ddl: + path: /clickhouse/task_queue/ddl + http_port: 8123 + interserver_http_port: 9009 + listen_host: 0.0.0.0 + logger: + console: 1 + count: 10 + formatting: + type: console + level: information + size: 1000M + macros: + replica: "01" + shard: "01" + profiles: + default: + load_balancing: random + log_queries: 1 + max_memory_usage: 10000000000 + quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 + user_directories: + users_xml: + path: users.xml + remote_servers: + cluster: + shard: + - replica: + host: signoz-telemetrystore-clickhouse + port: 9000 + zookeeper: + node: + - host: signoz-telemetrykeeper-zookeeper-0 + port: 9181 + tcp_port: 9000 + user_defined_executable_functions_config: '*function.yaml' + user_scripts_path: /var/lib/clickhouse/user_scripts/ + users: + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 + functions.yaml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable diff --git a/docs/examples/kubernetes/helm-patches/pours/deployment/values.yaml b/docs/examples/kubernetes/helm-patches/pours/deployment/values.yaml new file mode 100644 index 0000000..e88b6dc --- /dev/null +++ b/docs/examples/kubernetes/helm-patches/pours/deployment/values.yaml @@ -0,0 +1,96 @@ +clickhouse: + enabled: true + fullnameOverride: signoz-telemetrystore-clickhouse + image: + repository: clickhouse/clickhouse-server + tag: 25.5.6 + layout: + replicasCount: 0 + shardsCount: 1 + persistence: + enabled: true + size: 100Gi + storageClass: gp3 + resources: + limits: + cpu: "4" + memory: 8Gi + requests: + cpu: "2" + memory: 4Gi + zookeeper: + enabled: true + fullnameOverride: signoz-telemetrykeeper-zookeeper + image: + repository: signoz/zookeeper + tag: 3.7.1 + replicaCount: 1 +fullnameOverride: signoz +otelCollector: + enabled: true + image: + repository: signoz/signoz-otel-collector + tag: latest + name: ingester + ports: + otlp: + containerPort: 4317 + enabled: true + protocol: TCP + servicePort: 4317 + otlp-http: + containerPort: 4318 + enabled: true + protocol: TCP + servicePort: 4318 + replicaCount: 1 + resources: + limits: + cpu: "2" + memory: 4Gi + requests: + cpu: "1" + memory: 2Gi +postgresql: + auth: + database: signoz + password: signoz + username: signoz + enabled: true + fullnameOverride: signoz-metastore-postgres + image: + repository: postgres + tag: 16 +signoz: + env: + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@signoz-metastore-postgres:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://signoz-telemetrystore-clickhouse:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + image: + repository: signoz/signoz + tag: latest + name: signoz + nodeSelector: + node-role.kubernetes.io/observability: "" + replicaCount: 1 + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: 500m + memory: 1Gi + service: + internalPort: 8085 + opampPort: 4320 + port: 8080 + type: ClusterIP + tolerations: + - effect: NoSchedule + key: dedicated + operator: Equal + value: signoz +telemetryStoreMigrator: + enabled: true + name: signoz-telemetrystore-migrator diff --git a/docs/examples/kubernetes/helm/README.md b/docs/examples/kubernetes/helm/README.md new file mode 100644 index 0000000..9090003 --- /dev/null +++ b/docs/examples/kubernetes/helm/README.md @@ -0,0 +1,100 @@ +# Kubernetes with Helm + +| Field | Value | +| --- | --- | +| **Mode** | `kubernetes` | +| **Flavor** | `helm` | +| **Platform** | `-` | + +## Overview + +Generates a `values.yaml` for the SigNoz Helm chart. Foundry translates the casting spec into Helm values so you can deploy with a single `helm install`. + +## Prerequisites + +- Kubernetes cluster (1.24+) +- Helm 3.x +- SigNoz Helm chart repo (`https://charts.signoz.io`) + +## Configuration + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + flavor: helm + mode: kubernetes +``` + +## Deploy + +```bash +foundryctl cast -f casting.yaml +``` + +Or step by step: + +```bash +# Generate values.yaml +foundryctl forge -f casting.yaml + +# Add the SigNoz Helm repo +helm repo add signoz https://charts.signoz.io + +# Install with Helm +helm install signoz signoz/signoz -f pours/deployment/values.yaml -n signoz --create-namespace +``` + +> [!NOTE] +> `foundryctl cast` is idempotent. It detects whether a Helm release already exists and runs `helm upgrade` instead of `helm install` accordingly. + +## Generated output + +```text +pours/deployment/ + values.yaml +``` + +## After deployment + +```bash +# Check pod status +kubectl get pods -n signoz + +# Port-forward the SigNoz UI +kubectl port-forward svc/signoz -n signoz 8080:8080 +``` + +Open `http://localhost:8080` to access the SigNoz UI. + +## Customization + +To set resource limits, storage classes, or scheduling constraints on the generated `values.yaml`, use [patches](../../../concepts/patches.md). See the [helm-patches](../helm-patches/) example for a complete working configuration. + +## Annotations + +Optional annotations to override the default Helm chart source. These are not required for standard deployments. + +| Annotation | Default | Description | +| --- | --- | --- | +| `foundry.signoz.io/kubernetes-helm-casting-chart` | `signoz/signoz` | Helm chart reference | +| `foundry.signoz.io/kubernetes-helm-casting-repo-url` | `https://charts.signoz.io` | Helm chart repository URL | +| `foundry.signoz.io/kubernetes-helm-casting-repo-name` | `signoz` | Helm chart repository name | +| `foundry.signoz.io/kubernetes-helm-casting-forge-chart` | - | Set to `true` to download and bundle the chart locally during forge | + +Example with a custom chart repo: + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz + annotations: + foundry.signoz.io/kubernetes-helm-casting-repo-url: https://my-registry.example.com/charts + foundry.signoz.io/kubernetes-helm-casting-chart: my-registry/signoz +spec: + deployment: + flavor: helm + mode: kubernetes +``` diff --git a/docs/examples/kubernetes/helm/casting.yaml b/docs/examples/kubernetes/helm/casting.yaml new file mode 100644 index 0000000..b1f88da --- /dev/null +++ b/docs/examples/kubernetes/helm/casting.yaml @@ -0,0 +1,8 @@ +apiVersion: v1alpha1 +kind: Installation +metadata: + name: signoz +spec: + deployment: + flavor: helm + mode: kubernetes diff --git a/docs/examples/kubernetes/helm/casting.yaml.lock b/docs/examples/kubernetes/helm/casting.yaml.lock new file mode 100644 index 0000000..7a12988 --- /dev/null +++ b/docs/examples/kubernetes/helm/casting.yaml.lock @@ -0,0 +1,645 @@ +apiVersion: v1alpha1 +kind: Installation +metadata: + name: signoz +spec: + deployment: + flavor: helm + mode: kubernetes + ingester: + spec: + cluster: + replicas: 1 + config: + data: + ingester.yaml: | + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 5s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://signoz-telemetrystore-clickhouse:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://signoz-telemetrystore-clickhouse:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://signoz-telemetrystore-clickhouse:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://signoz-telemetrystore-clickhouse:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + metadataexporter: + enabled: true + dsn: tcp://signoz-telemetrystore-clickhouse:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + - metadataexporter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + - metadataexporter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + - metadataexporter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: | + server_endpoint: tcp://signoz:4320/v1/opamp + enabled: true + env: + SIGNOZ_OTEL_COLLECTOR_TIMEOUT: 10m + image: signoz/signoz-otel-collector:latest + version: latest + status: + addresses: + otlp: null + config: + data: + ingester.yaml: | + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 5s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://signoz-telemetrystore-clickhouse:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://signoz-telemetrystore-clickhouse:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://signoz-telemetrystore-clickhouse:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://signoz-telemetrystore-clickhouse:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + metadataexporter: + enabled: true + dsn: tcp://signoz-telemetrystore-clickhouse:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + - metadataexporter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + - metadataexporter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + - metadataexporter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: | + server_endpoint: tcp://signoz:4320/v1/opamp + env: + SIGNOZ_OTEL_COLLECTOR_TIMEOUT: 10m + metastore: + kind: postgres + spec: + cluster: + replicas: 1 + config: {} + enabled: true + env: + POSTGRES_DB: signoz + POSTGRES_PASSWORD: signoz + POSTGRES_USER: signoz + image: postgres:16 + version: "16" + status: + addresses: + dsn: + - postgres://signoz-metastore-postgres:5432 + config: {} + env: + POSTGRES_DB: signoz + POSTGRES_PASSWORD: signoz + POSTGRES_USER: signoz + signoz: + spec: + cluster: + replicas: 1 + config: {} + enabled: true + env: + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@signoz-metastore-postgres:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://signoz-telemetrystore-clickhouse:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + image: signoz/signoz:latest + version: latest + status: + addresses: + apiserver: null + opamp: + - tcp://signoz:4320 + config: {} + env: + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@signoz-metastore-postgres:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://signoz-telemetrystore-clickhouse:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + telemetrykeeper: + kind: clickhousekeeper + spec: + cluster: + replicas: 1 + config: + data: + keeper-0.yaml: | + listen_host: 0.0.0.0 + logger: + level: information + console: true + keeper_server: + four_letter_word_white_list: "*" + coordination_settings: + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: signoz-telemetrykeeper-zookeeper-0 + port: 9234 + id: 0 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 + enabled: true + image: clickhouse/clickhouse-keeper:25.5.6 + version: 25.5.6 + status: + addresses: + client: + - tcp://signoz-telemetrykeeper-zookeeper-0:9181 + raft: + - tcp://signoz-telemetrykeeper-zookeeper-0:9234 + config: + data: + keeper-0.yaml: | + listen_host: 0.0.0.0 + logger: + level: information + console: true + keeper_server: + four_letter_word_white_list: "*" + coordination_settings: + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: signoz-telemetrykeeper-zookeeper-0 + port: 9234 + id: 0 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 + telemetrystore: + kind: clickhouse + spec: + cluster: + replicas: 0 + shards: 1 + config: + data: + config-0-0.yaml: | + path: /var/lib/clickhouse/ + tmp_path: /var/lib/clickhouse/tmp/ + user_files_path: /var/lib/clickhouse/user_files/ + format_schema_path: /var/lib/clickhouse/format_schemas/ + dictionaries_config: '*_dictionary.xml' + display_name: cluster + distributed_ddl: + path: /clickhouse/task_queue/ddl + http_port: 8123 + interserver_http_port: 9009 + listen_host: 0.0.0.0 + logger: + console: 1 + count: 10 + formatting: + type: console + level: information + size: 1000M + macros: + replica: "00" + shard: "00" + profiles: + default: + allow_simdjson: 0 + load_balancing: random + log_queries: 1 + quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 + user_directories: + users_xml: + path: users.xml + remote_servers: + cluster: + shard: + - replica: + - host: signoz-telemetrystore-clickhouse + port: 9000 + zookeeper: + node: + - host: signoz-telemetrykeeper-zookeeper-0 + port: 9181 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_thread_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_views_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + part_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + metric_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + asynchronous_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + trace_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + error_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + latency_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + session_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + text_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + zookeeper_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + tcp_port: 9000 + user_defined_executable_functions_config: '*function.yaml' + user_scripts_path: /var/lib/clickhouse/user_scripts/ + users: + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 + functions.yaml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable + enabled: true + image: clickhouse/clickhouse-server:25.5.6 + version: 25.5.6 + status: + addresses: + tcp: + - tcp://signoz-telemetrystore-clickhouse:9000 + config: + data: + config-0-0.yaml: | + path: /var/lib/clickhouse/ + tmp_path: /var/lib/clickhouse/tmp/ + user_files_path: /var/lib/clickhouse/user_files/ + format_schema_path: /var/lib/clickhouse/format_schemas/ + dictionaries_config: '*_dictionary.xml' + display_name: cluster + distributed_ddl: + path: /clickhouse/task_queue/ddl + http_port: 8123 + interserver_http_port: 9009 + listen_host: 0.0.0.0 + logger: + console: 1 + count: 10 + formatting: + type: console + level: information + size: 1000M + macros: + replica: "00" + shard: "00" + profiles: + default: + allow_simdjson: 0 + load_balancing: random + log_queries: 1 + quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 + user_directories: + users_xml: + path: users.xml + remote_servers: + cluster: + shard: + - replica: + - host: signoz-telemetrystore-clickhouse + port: 9000 + zookeeper: + node: + - host: signoz-telemetrykeeper-zookeeper-0 + port: 9181 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_thread_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_views_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + part_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + metric_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + asynchronous_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + trace_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + error_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + latency_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + session_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + text_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + zookeeper_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + tcp_port: 9000 + user_defined_executable_functions_config: '*function.yaml' + user_scripts_path: /var/lib/clickhouse/user_scripts/ + users: + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 + functions.yaml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable diff --git a/docs/examples/kubernetes/helm/pours/deployment/values.yaml b/docs/examples/kubernetes/helm/pours/deployment/values.yaml new file mode 100644 index 0000000..24a2bb0 --- /dev/null +++ b/docs/examples/kubernetes/helm/pours/deployment/values.yaml @@ -0,0 +1,68 @@ +clickhouse: + enabled: true + fullnameOverride: signoz-telemetrystore-clickhouse + image: + repository: clickhouse/clickhouse-server + tag: 25.5.6 + layout: + replicasCount: 0 + shardsCount: 1 + zookeeper: + enabled: true + fullnameOverride: signoz-telemetrykeeper-zookeeper + image: + repository: signoz/zookeeper + tag: 3.7.1 + replicaCount: 1 +fullnameOverride: signoz +otelCollector: + enabled: true + env: + SIGNOZ_OTEL_COLLECTOR_TIMEOUT: 10m + image: + repository: signoz/signoz-otel-collector + tag: latest + name: ingester + ports: + otlp: + containerPort: 4317 + enabled: true + protocol: TCP + servicePort: 4317 + otlp-http: + containerPort: 4318 + enabled: true + protocol: TCP + servicePort: 4318 + replicaCount: 1 +postgresql: + auth: + database: signoz + password: signoz + username: signoz + enabled: true + fullnameOverride: signoz-metastore-postgres + image: + repository: postgres + tag: 16 +signoz: + env: + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@signoz-metastore-postgres:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://signoz-telemetrystore-clickhouse:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + image: + repository: signoz/signoz + tag: latest + name: signoz + replicaCount: 1 + service: + internalPort: 8085 + opampPort: 4320 + port: 8080 + type: ClusterIP +telemetryStoreMigrator: + enabled: true + env: + SIGNOZ_OTEL_COLLECTOR_TIMEOUT: 10m + name: signoz-telemetrystore-migrator diff --git a/docs/examples/kubernetes/kustomize-patches/README.md b/docs/examples/kubernetes/kustomize-patches/README.md new file mode 100644 index 0000000..646ce6d --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/README.md @@ -0,0 +1,159 @@ +# Kustomize with Patches + +| Field | Value | +| --- | --- | +| **Mode** | `kubernetes` | +| **Flavor** | `kustomize` | +| **Platform** | `-` | + +## Overview + +Extends the base [kubernetes/kustomize](../kustomize/) example with production patches for AWS. Demonstrates setting storage classes, resource limits, dedicated node scheduling, and a LoadBalancer service using `spec.patches`. + +For patch reference, see [Patches](../../../concepts/patches.md). + +## Prerequisites + +- Kubernetes cluster (1.24+) +- `kubectl` with kustomize support +- Cloud provider storage class (for example, `gp3` on AWS) + +## Configuration + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + flavor: kustomize + mode: kubernetes + patches: + # Storage class and size on ClickHouse data volumes + - target: "deployment/telemetrystore/clickhouse/clickhouseinstallation.yaml" + operations: + - op: add + path: /spec/templates/volumeClaimTemplates/0/spec/storageClassName + value: gp3 + - op: replace + path: /spec/templates/volumeClaimTemplates/0/spec/resources/requests/storage + value: 100Gi + - op: add + path: /spec/templates/volumeClaimTemplates/1/spec/storageClassName + value: gp3 + + # Resource limits on ClickHouse + - target: "deployment/telemetrystore/clickhouse/clickhouseinstallation.yaml" + operations: + - op: replace + path: /spec/templates/podTemplates/0/spec/containers/0/resources + value: + requests: + cpu: "2" + memory: "4Gi" + limits: + cpu: "4" + memory: "8Gi" + + # Resource limits on SigNoz + - target: "deployment/signoz/statefulset.yaml" + operations: + - op: replace + path: /spec/template/spec/containers/0/resources + value: + requests: + memory: "1Gi" + cpu: "500m" + limits: + memory: "2Gi" + cpu: "1" + + # Tolerations and nodeSelector for dedicated nodes + - target: "deployment/signoz/statefulset.yaml" + operations: + - op: add + path: /spec/template/spec/tolerations + value: + - key: "dedicated" + operator: "Equal" + value: "signoz" + effect: "NoSchedule" + - op: add + path: /spec/template/spec/nodeSelector + value: + node-role.kubernetes.io/observability: "" + + # LoadBalancer with AWS NLB + - target: "deployment/signoz/service.yaml" + operations: + - op: replace + path: /spec/type + value: LoadBalancer + - op: add + path: /metadata/annotations + value: + service.beta.kubernetes.io/aws-load-balancer-type: nlb + + # Resource limits on ingester + - target: "deployment/ingester/deployment.yaml" + operations: + - op: replace + path: /spec/template/spec/containers/0/resources + value: + requests: + cpu: "1" + memory: "2Gi" + limits: + cpu: "2" + memory: "4Gi" + + # Storage class on PostgreSQL metastore + - target: "deployment/metastore/postgres/statefulset.yaml" + operations: + - op: add + path: /spec/volumeClaimTemplates/0/spec/storageClassName + value: gp3 + - op: replace + path: /spec/volumeClaimTemplates/0/spec/resources/requests/storage + value: 20Gi +``` + +## Deploy + +```bash +foundryctl cast -f casting.yaml +``` + +Or step by step: + +```bash +foundryctl forge -f casting.yaml +kubectl apply -k pours/deployment/ +``` + +## Generated output + +Same structure as [kubernetes/kustomize](../kustomize/), with patches applied to the generated manifests. + +## Customization + +### Storage class and size + +Sets the storage class on volume claim templates. Adjust for your cloud provider: `gp3` on AWS, `pd-ssd` on GCP, `managed-premium` on Azure. + +ClickHouse has two volume claim templates (`data-0` and `default`). Both should use the same storage class. + +### Resource limits + +Sets CPU and memory requests/limits per component. The JSON Patch path varies by resource kind: + +- ClickHouseInstallation: `/spec/templates/podTemplates/0/spec/containers/0/resources` +- StatefulSet or Deployment: `/spec/template/spec/containers/0/resources` + +### Tolerations and nodeSelector + +Schedules SigNoz on nodes labeled `node-role.kubernetes.io/observability` with a `dedicated=signoz:NoSchedule` toleration. + +### LoadBalancer with AWS NLB + +Changes the SigNoz Service to `LoadBalancer` with `service.beta.kubernetes.io/aws-load-balancer-type: nlb`. diff --git a/docs/examples/kubernetes/kustomize-patches/casting.yaml b/docs/examples/kubernetes/kustomize-patches/casting.yaml new file mode 100644 index 0000000..c988250 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/casting.yaml @@ -0,0 +1,96 @@ +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + flavor: kustomize + mode: kubernetes + patches: + # 1. Set storage class and size on ClickHouse data volumes + # Required for cloud providers (gp3 on AWS, pd-ssd on GCP, managed-premium on Azure) + - target: "deployment/telemetrystore/clickhouse/clickhouseinstallation.yaml" + operations: + - op: add + path: /spec/templates/volumeClaimTemplates/0/spec/storageClassName + value: gp3 + - op: replace + path: /spec/templates/volumeClaimTemplates/0/spec/resources/requests/storage + value: 100Gi + - op: add + path: /spec/templates/volumeClaimTemplates/1/spec/storageClassName + value: gp3 + + # 2. Set resource limits on ClickHouse for production sizing + - target: "deployment/telemetrystore/clickhouse/clickhouseinstallation.yaml" + operations: + - op: replace + path: /spec/templates/podTemplates/0/spec/containers/0/resources + value: + requests: + cpu: "2" + memory: "4Gi" + limits: + cpu: "4" + memory: "8Gi" + + # 3. Set resource limits and requests on SigNoz + - target: "deployment/signoz/statefulset.yaml" + operations: + - op: replace + path: /spec/template/spec/containers/0/resources + value: + requests: + memory: "1Gi" + cpu: "500m" + limits: + memory: "2Gi" + cpu: "1" + + # 4. Add tolerations and nodeSelector for dedicated observability nodes + - target: "deployment/signoz/statefulset.yaml" + operations: + - op: add + path: /spec/template/spec/tolerations + value: + - key: "dedicated" + operator: "Equal" + value: "signoz" + effect: "NoSchedule" + - op: add + path: /spec/template/spec/nodeSelector + value: + node-role.kubernetes.io/observability: "" + + # 5. Change SigNoz service to LoadBalancer with AWS NLB + - target: "deployment/signoz/service.yaml" + operations: + - op: replace + path: /spec/type + value: LoadBalancer + - op: add + path: /metadata/annotations + value: + service.beta.kubernetes.io/aws-load-balancer-type: nlb + + # 6. Set resource limits on ingester + - target: "deployment/ingester/deployment.yaml" + operations: + - op: replace + path: /spec/template/spec/containers/0/resources + value: + requests: + cpu: "1" + memory: "2Gi" + limits: + cpu: "2" + memory: "4Gi" + + # 7. Set storage class on PostgreSQL metastore + - target: "deployment/metastore/postgres/statefulset.yaml" + operations: + - op: add + path: /spec/volumeClaimTemplates/0/spec/storageClassName + value: gp3 + - op: replace + path: /spec/volumeClaimTemplates/0/spec/resources/requests/storage + value: 20Gi diff --git a/docs/examples/kubernetes/kustomize-patches/casting.yaml.lock b/docs/examples/kubernetes/kustomize-patches/casting.yaml.lock new file mode 100644 index 0000000..3e83f4a --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/casting.yaml.lock @@ -0,0 +1,819 @@ +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + flavor: kustomize + mode: kubernetes + ingester: + spec: + cluster: + replicas: 1 + config: + data: + ingester.yaml: | + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 1s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 1s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://signoz-clickhouse:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://signoz-clickhouse:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://signoz-clickhouse:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://signoz-clickhouse:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: | + server_endpoint: ws://signoz-signoz:4320/v1/opamp + enabled: true + image: signoz/signoz-otel-collector:latest + version: latest + status: + addresses: + otlp: null + config: + data: + ingester.yaml: | + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 1s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 1s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://signoz-clickhouse:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://signoz-clickhouse:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://signoz-clickhouse:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://signoz-clickhouse:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: | + server_endpoint: ws://signoz-signoz:4320/v1/opamp + metastore: + kind: postgres + spec: + cluster: + replicas: 1 + config: {} + enabled: true + env: + POSTGRES_DB: signoz + POSTGRES_PASSWORD: signoz + POSTGRES_USER: signoz + image: postgres:16 + version: "16" + status: + addresses: + dsn: + - postgres://signoz-metastore:5432 + config: {} + env: + POSTGRES_DB: signoz + POSTGRES_PASSWORD: signoz + POSTGRES_USER: signoz + patches: + - operations: + - op: add + path: /spec/templates/volumeClaimTemplates/0/spec/storageClassName + value: gp3 + - op: replace + path: /spec/templates/volumeClaimTemplates/0/spec/resources/requests/storage + value: 100Gi + - op: add + path: /spec/templates/volumeClaimTemplates/1/spec/storageClassName + value: gp3 + target: deployment/telemetrystore/clickhouse/clickhouseinstallation.yaml + - operations: + - op: replace + path: /spec/templates/podTemplates/0/spec/containers/0/resources + value: + limits: + cpu: "4" + memory: 8Gi + requests: + cpu: "2" + memory: 4Gi + target: deployment/telemetrystore/clickhouse/clickhouseinstallation.yaml + - operations: + - op: replace + path: /spec/template/spec/containers/0/resources + value: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: 500m + memory: 1Gi + target: deployment/signoz/statefulset.yaml + - operations: + - op: add + path: /spec/template/spec/tolerations + value: + - effect: NoSchedule + key: dedicated + operator: Equal + value: signoz + - op: add + path: /spec/template/spec/nodeSelector + value: + node-role.kubernetes.io/observability: "" + target: deployment/signoz/statefulset.yaml + - operations: + - op: replace + path: /spec/type + value: LoadBalancer + - op: add + path: /metadata/annotations + value: + service.beta.kubernetes.io/aws-load-balancer-type: nlb + target: deployment/signoz/service.yaml + - operations: + - op: replace + path: /spec/template/spec/containers/0/resources + value: + limits: + cpu: "2" + memory: 4Gi + requests: + cpu: "1" + memory: 2Gi + target: deployment/ingester/deployment.yaml + - operations: + - op: add + path: /spec/volumeClaimTemplates/0/spec/storageClassName + value: gp3 + - op: replace + path: /spec/volumeClaimTemplates/0/spec/resources/requests/storage + value: 20Gi + target: deployment/metastore/postgres/statefulset.yaml + signoz: + spec: + cluster: + replicas: 1 + config: {} + enabled: true + env: + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@signoz-metastore:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://signoz-clickhouse:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + image: signoz/signoz:latest + version: latest + status: + addresses: + apiserver: null + opamp: + - ws://signoz-signoz:4320 + config: {} + env: + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@signoz-metastore:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://signoz-clickhouse:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + telemetrykeeper: + kind: clickhousekeeper + spec: + cluster: + replicas: 1 + config: + data: + keeper-0.yaml: | + listen_host: 0.0.0.0 + logger: + level: information + console: true + keeper_server: + four_letter_word_white_list: "*" + coordination_settings: + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: signoz-clickhouse-keeper-0 + port: 9234 + id: 0 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 + enabled: true + image: clickhouse/clickhouse-keeper:25.5.6 + version: 25.5.6 + status: + addresses: + client: + - tcp://signoz-clickhouse-keeper-0:9181 + raft: + - tcp://signoz-clickhouse-keeper-0:9234 + config: + data: + keeper-0.yaml: | + listen_host: 0.0.0.0 + logger: + level: information + console: true + keeper_server: + four_letter_word_white_list: "*" + coordination_settings: + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: signoz-clickhouse-keeper-0 + port: 9234 + id: 0 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 + telemetrystore: + kind: clickhouse + spec: + cluster: + replicas: 0 + shards: 1 + config: + data: + config.yaml: | + async_load_databases: false + dictionaries_config: '*_dictionary.xml' + display_name: cluster + distributed_ddl: + path: /clickhouse/task_queue/ddl + error_log: + ttl: event_date + INTERVAL 7 DAY DELETE + format_schema_path: /etc/clickhouse-server/config.d/ + http_port: 8123 + interserver_http_port: 9009 + latency_log: + ttl: event_date + INTERVAL 3 DAY DELETE + listen_host: 0.0.0.0 + logger: + console: 1 + count: 10 + formatting: + names: + date_time: date_time + level: level + logger_name: logger_name + message: message + query_id: query_id + source_file: source_file + source_line: source_line + thread_id: thread_id + thread_name: thread_name + type: json + level: information + size: 1000M + macros: + replica: "01" + shard: "01" + merge_tree: + storage_policy: data + path: /var/lib/clickhouse/ + profiles: + admin: + max_threads: 16 + query_plan_max_limit_for_lazy_materialization: "0" + secondary_indices_enable_bulk_filtering: "0" + default: + allow_experimental_window_functions: "1" + allow_nondeterministic_mutations: "1" + load_balancing: random + log_queries: 1 + max_memory_usage: 10000000000 + max_threads: 16 + query_plan_max_limit_for_lazy_materialization: "0" + secondary_indices_enable_bulk_filtering: "0" + prometheus: + endpoint: /metrics + port: 9363 + query_metric_log: + ttl: event_date + INTERVAL 3 DAY DELETE + quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 + remote_servers: + cluster: + shard: + - replica: + host: signoz-clickhouse + port: 9000 + send_crash_reports: + enabled: false + storage_configuration: + disks: + data0: + keep_free_space_bytes: 10485760 + path: /mnt/data-0/ + default: {} + policies: + data: + volumes: + data0: + disk: data0 + default: + volumes: + default: + disk: default + tcp_port: 9000 + text_log: + ttl: event_date + INTERVAL 3 DAY DELETE + tmp_path: /var/lib/clickhouse/tmp/ + user_defined_executable_functions_config: /etc/clickhouse-server/functions/custom-functions.xml + user_directories: + users_xml: + path: users.xml + user_files_path: /var/lib/clickhouse/user_files/ + user_scripts_path: /var/lib/clickhouse/user_scripts/ + users: + admin: + networks: + ip: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 172.16.0.0/12 + - 192.0.0.0/24 + - 198.18.0.0/15 + - 192.168.0.0/16 + password: 27ff0399-0d3a-4bd8-919d-17c2181e6fb9 + profile: default + quota: default + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 + zookeeper: + node: + - host: signoz-clickhouse-keeper-0 + port: 9181 + functions.yaml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable + enabled: true + image: clickhouse/clickhouse-server:25.5.6 + version: 25.5.6 + status: + addresses: + tcp: + - tcp://signoz-clickhouse:9000 + config: + data: + config.yaml: | + async_load_databases: false + dictionaries_config: '*_dictionary.xml' + display_name: cluster + distributed_ddl: + path: /clickhouse/task_queue/ddl + error_log: + ttl: event_date + INTERVAL 7 DAY DELETE + format_schema_path: /etc/clickhouse-server/config.d/ + http_port: 8123 + interserver_http_port: 9009 + latency_log: + ttl: event_date + INTERVAL 3 DAY DELETE + listen_host: 0.0.0.0 + logger: + console: 1 + count: 10 + formatting: + names: + date_time: date_time + level: level + logger_name: logger_name + message: message + query_id: query_id + source_file: source_file + source_line: source_line + thread_id: thread_id + thread_name: thread_name + type: json + level: information + size: 1000M + macros: + replica: "01" + shard: "01" + merge_tree: + storage_policy: data + path: /var/lib/clickhouse/ + profiles: + admin: + max_threads: 16 + query_plan_max_limit_for_lazy_materialization: "0" + secondary_indices_enable_bulk_filtering: "0" + default: + allow_experimental_window_functions: "1" + allow_nondeterministic_mutations: "1" + load_balancing: random + log_queries: 1 + max_memory_usage: 10000000000 + max_threads: 16 + query_plan_max_limit_for_lazy_materialization: "0" + secondary_indices_enable_bulk_filtering: "0" + prometheus: + endpoint: /metrics + port: 9363 + query_metric_log: + ttl: event_date + INTERVAL 3 DAY DELETE + quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 + remote_servers: + cluster: + shard: + - replica: + host: signoz-clickhouse + port: 9000 + send_crash_reports: + enabled: false + storage_configuration: + disks: + data0: + keep_free_space_bytes: 10485760 + path: /mnt/data-0/ + default: {} + policies: + data: + volumes: + data0: + disk: data0 + default: + volumes: + default: + disk: default + tcp_port: 9000 + text_log: + ttl: event_date + INTERVAL 3 DAY DELETE + tmp_path: /var/lib/clickhouse/tmp/ + user_defined_executable_functions_config: /etc/clickhouse-server/functions/custom-functions.xml + user_directories: + users_xml: + path: users.xml + user_files_path: /var/lib/clickhouse/user_files/ + user_scripts_path: /var/lib/clickhouse/user_scripts/ + users: + admin: + networks: + ip: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 172.16.0.0/12 + - 192.0.0.0/24 + - 198.18.0.0/15 + - 192.168.0.0/16 + password: 27ff0399-0d3a-4bd8-919d-17c2181e6fb9 + profile: default + quota: default + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 + zookeeper: + node: + - host: signoz-clickhouse-keeper-0 + port: 9181 + functions.yaml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable + extras: + _overrides: | + async_load_databases: false + error_log: + ttl: event_date + INTERVAL 7 DAY DELETE + format_schema_path: /etc/clickhouse-server/config.d/ + latency_log: + ttl: event_date + INTERVAL 3 DAY DELETE + logger: + formatting: + names: + date_time: date_time + level: level + logger_name: logger_name + message: message + query_id: query_id + source_file: source_file + source_line: source_line + thread_id: thread_id + thread_name: thread_name + type: json + merge_tree: + storage_policy: data + profiles: + admin: + max_threads: 16 + query_plan_max_limit_for_lazy_materialization: "0" + secondary_indices_enable_bulk_filtering: "0" + default: + allow_experimental_window_functions: "1" + allow_nondeterministic_mutations: "1" + max_threads: 16 + query_plan_max_limit_for_lazy_materialization: "0" + secondary_indices_enable_bulk_filtering: "0" + prometheus: + endpoint: /metrics + port: 9363 + query_metric_log: + ttl: event_date + INTERVAL 3 DAY DELETE + send_crash_reports: + enabled: false + storage_configuration: + disks: + data0: + keep_free_space_bytes: 10485760 + path: /mnt/data-0/ + default: {} + policies: + data: + volumes: + data0: + disk: data0 + default: + volumes: + default: + disk: default + text_log: + ttl: event_date + INTERVAL 3 DAY DELETE + user_defined_executable_functions_config: /etc/clickhouse-server/functions/custom-functions.xml + users: + admin: + networks: + ip: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 172.16.0.0/12 + - 192.0.0.0/24 + - 198.18.0.0/15 + - 192.168.0.0/16 + password: 27ff0399-0d3a-4bd8-919d-17c2181e6fb9 + profile: default + quota: default diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/clusterrole.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/clusterrole.yaml new file mode 100644 index 0000000..4cbeaa5 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/clusterrole.yaml @@ -0,0 +1,184 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator +rules: +- apiGroups: + - "" + resources: + - configmaps + - services + - persistentvolumeclaims + - secrets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch + - delete +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete +- apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - patch + - update + - delete +- apiGroups: + - apps + resourceNames: + - clickhouse-operator + resources: + - deployments + verbs: + - get + - patch + - update + - delete +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list +- apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations + verbs: + - get + - list + - watch + - patch + - update + - delete +- apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallationtemplates + - clickhouseoperatorconfigurations + verbs: + - get + - list + - watch +- apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/finalizers + - clickhouseinstallationtemplates/finalizers + - clickhouseoperatorconfigurations/finalizers + verbs: + - update +- apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/status + - clickhouseinstallationtemplates/status + - clickhouseoperatorconfigurations/status + verbs: + - get + - update + - patch + - create + - delete +- apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations + verbs: + - get + - list + - watch + - patch + - update + - delete +- apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/finalizers + verbs: + - update +- apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/status + verbs: + - get + - update + - patch + - create + - delete diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/clusterrolebinding.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/clusterrolebinding.yaml new file mode 100644 index 0000000..baf43b7 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: signoz-telemetrystore-clickhouse-operator +subjects: +- kind: ServiceAccount + name: signoz-telemetrystore-clickhouse-operator + namespace: signoz diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/configmap.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/configmap.yaml new file mode 100644 index 0000000..ec74edf --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/configmap.yaml @@ -0,0 +1,576 @@ +apiVersion: v1 +data: null +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-etc-confd-files +--- +apiVersion: v1 +data: + 01-clickhouse-01-listen.xml: | + <clickhouse> + <!-- Listen wildcard address to allow accepting connections from other containers and host network. --> + <listen_host>::</listen_host> + <listen_host>0.0.0.0</listen_host> + <listen_try>1</listen_try> + </clickhouse> + 01-clickhouse-02-logger.xml: | + <clickhouse> + <logger> + <!-- Possible levels: https://github.com/pocoproject/poco/blob/devel/Foundation/include/Poco/Logger.h#L105 --> + <level>warning</level> + <formatting> + <type>json</type> + </formatting> + <log>/var/log/clickhouse-server/clickhouse-server.log</log> + <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog> + <size>1000M</size> + <count>10</count> + <!-- Default behavior is autodetection (log to console if not daemon mode and is tty) --> + <console>1</console> + </logger> + </clickhouse> + 01-clickhouse-03-query_log.xml: | + <clickhouse> + <query_log replace="1"> + <database>system</database> + <table>query_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </query_log> + </clickhouse> + 01-clickhouse-04-part_log.xml: | + <clickhouse> + <part_log replace="1"> + <database>system</database> + <table>part_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </part_log> + </clickhouse> + 01-clickhouse-05-trace_log.xml: |- + <clickhouse> + <trace_log replace="1"> + <database>system</database> + <table>trace_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </trace_log> + </clickhouse> + 01-clickhouse-06-asynchronous_insert_log.xml: | + <clickhouse> + <asynchronous_insert_log replace="1"> + <database>system</database> + <table>asynchronous_insert_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </asynchronous_insert_log> + </clickhouse> + 01-clickhouse-07-asynchronous_metric_log.xml: | + <clickhouse> + <asynchronous_metric_log replace="1"> + <database>system</database> + <table>asynchronous_metric_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </asynchronous_metric_log> + </clickhouse> + 01-clickhouse-08-backup_log.xml: | + <clickhouse> + <backup_log replace="1"> + <database>system</database> + <table>backup_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </backup_log> + </clickhouse> + 01-clickhouse-09-blob_storage_log.xml: | + <clickhouse> + <blob_storage_log replace="1"> + <database>system</database> + <table>blob_storage_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </blob_storage_log> + </clickhouse> + 01-clickhouse-10-crash_log.xml: | + <clickhouse> + <crash_log replace="1"> + <database>system</database> + <table>crash_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 14 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </crash_log> + </clickhouse> + 01-clickhouse-11-metric_log.xml: | + <clickhouse> + <metric_log replace="1"> + <database>system</database> + <table>metric_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </metric_log> + </clickhouse> + 01-clickhouse-12-query_thread_log.xml: | + <clickhouse> + <query_thread_log replace="1"> + <database>system</database> + <table>query_thread_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </query_thread_log> + </clickhouse> + 01-clickhouse-13-query_views_log.xml: | + <clickhouse> + <query_views_log replace="1"> + <database>system</database> + <table>query_views_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </query_views_log> + </clickhouse> + 01-clickhouse-14-session_log.xml: | + <clickhouse> + <session_log replace="1"> + <database>system</database> + <table>session_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </session_log> + </clickhouse> + 01-clickhouse-15-zookeeper_log.xml: | + <clickhouse> + <zookeeper_log replace="1"> + <database>system</database> + <table>zookeeper_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </zookeeper_log> + </clickhouse> + 01-clickhouse-16-processors_profile_log.xml: | + <clickhouse> + <processors_profile_log replace="1"> + <database>system</database> + <table>processors_profile_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </processors_profile_log> + </clickhouse> +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-etc-configd-files +--- +apiVersion: v1 +data: + config.yaml: |- + annotation: + exclude: + - argocd.argoproj.io/tracking-id + include: [] + clickhouse: + access: + password: "" + port: 8123 + rootCA: "" + scheme: auto + secret: + name: "clickhouse-operator" + namespace: "signoz" + timeouts: + connect: 1 + query: 4 + username: "" + addons: + rules: + - spec: + configuration: + files: null + profiles: null + quotas: null + settings: null + users: null + version: '*' + - spec: + configuration: + files: null + profiles: null + quotas: null + settings: null + users: + '{clickhouseOperatorUser}/access_management': 1 + '{clickhouseOperatorUser}/named_collection_control': 1 + '{clickhouseOperatorUser}/show_named_collections': 1 + '{clickhouseOperatorUser}/show_named_collections_secrets': 1 + version: '>= 23.3' + - spec: + configuration: + files: null + profiles: + clickhouse_operator/format_display_secrets_in_show_and_select: 1 + quotas: null + settings: + display_secrets_in_show_and_select: 1 + users: null + version: '>= 23.5' + configuration: + file: + path: + common: chi/config.d + host: chi/conf.d + user: chi/users.d + network: + hostRegexpTemplate: (chi-{chi}-[^.]+\d+-\d+|clickhouse\-{chi})\.{namespace}\.svc\.cluster\.local$ + user: + default: + networksIP: + - ::1 + - 127.0.0.1 + password: default + profile: default + quota: default + configurationRestartPolicy: + rules: + - rules: + - settings/*: "yes" + - settings/access_control_path: "no" + - settings/dictionaries_config: "no" + - settings/max_server_memory_*: "no" + - settings/max_*_to_drop: "no" + - settings/max_concurrent_queries: "no" + - settings/models_config: "no" + - settings/user_defined_executable_functions_config: "no" + - settings/logger/*: "no" + - settings/macros/*: "no" + - settings/remote_servers/*: "no" + - settings/user_directories/*: "no" + - settings/display_secrets_in_show_and_select: "no" + - zookeeper/*: "yes" + - files/*.xml: "yes" + - files/config.d/*.xml: "yes" + - files/config.d/*dict*.xml: "no" + - files/config.d/*no_restart*: "no" + - profiles/default/background_*_pool_size: "yes" + - profiles/default/max_*_for_server: "yes" + version: '*' + - rules: + - settings/logger: "yes" + version: 21.* + metrics: + timeouts: + collect: 9 + keeper: + configuration: + file: + path: + common: chk/keeper_config.d + host: chk/conf.d + user: chk/users.d + label: + appendScope: "no" + exclude: [] + include: [] + logger: + alsologtostderr: "false" + log_backtrace_at: "" + logtostderr: "true" + stderrthreshold: "" + v: "1" + vmodule: "" + metrics: + labels: + exclude: [] + pod: + terminationGracePeriod: 30 + reconcile: + host: + wait: + exclude: true + include: false + queries: true + replicas: + all: false + delay: 10 + new: true + runtime: + reconcileCHIsThreadsNumber: 10 + reconcileShardsMaxConcurrencyPercent: 50 + reconcileShardsThreadsNumber: 5 + statefulSet: + create: + onFailure: ignore + update: + onFailure: abort + pollInterval: 5 + timeout: 300 + statefulSet: + revisionHistoryLimit: 0 + status: + fields: + action: false + actions: false + error: false + errors: false + template: + chi: + path: chi/templates.d + policy: ApplyOnNextReconcile + chk: + path: chk/templates.d + policy: ApplyOnNextReconcile + watch: + namespaces: + - ".*" +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-etc-files +--- +apiVersion: v1 +data: + 001-templates.json.example: | + { + "apiVersion": "clickhouse.altinity.com/v1", + "kind": "ClickHouseInstallationTemplate", + "metadata": { + "name": "01-default-volumeclaimtemplate" + }, + "spec": { + "templates": { + "volumeClaimTemplates": [ + { + "name": "chi-default-volume-claim-template", + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "2Gi" + } + } + } + } + ], + "podTemplates": [ + { + "name": "chi-default-oneperhost-pod-template", + "distribution": "OnePerHost", + "spec": { + "containers" : [ + { + "name": "clickhouse", + "image": "clickhouse/clickhouse-server:23.8", + "ports": [ + { + "name": "http", + "containerPort": 8123 + }, + { + "name": "client", + "containerPort": 9000 + }, + { + "name": "interserver", + "containerPort": 9009 + } + ] + } + ] + } + } + ] + } + } + } + default-pod-template.yaml.example: | + apiVersion: "clickhouse.altinity.com/v1" + kind: "ClickHouseInstallationTemplate" + metadata: + name: "default-oneperhost-pod-template" + spec: + templates: + podTemplates: + - name: default-oneperhost-pod-template + distribution: "OnePerHost" + default-storage-template.yaml.example: | + apiVersion: "clickhouse.altinity.com/v1" + kind: "ClickHouseInstallationTemplate" + metadata: + name: "default-storage-template-2Gi" + spec: + templates: + volumeClaimTemplates: + - name: default-storage-template-2Gi + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + readme: Templates in this folder are packaged with an operator and available via 'useTemplate' +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-etc-templatesd-files +--- +apiVersion: v1 +data: + 01-clickhouse-user.xml: | + <yandex> + <users> + <clickhouse_operator> + <networks> + <ip>127.0.0.1</ip> + </networks> + <password_sha256_hex></password_sha256_hex> + <profile>clickhouse_operator</profile> + <quota>default</quota> + </clickhouse_operator> + </users> + <profiles> + <clickhouse_operator> + <log_queries>0</log_queries> + <skip_unavailable_shards>1</skip_unavailable_shards> + <http_connection_timeout>10</http_connection_timeout> + </clickhouse_operator> + </profiles> + </yandex> + 02-clickhouse-default-profile.xml: | + <yandex> + <profiles> + <default> + <log_queries>1</log_queries> + <connect_timeout_with_failover_ms>1000</connect_timeout_with_failover_ms> + <distributed_aggregation_memory_efficient>1</distributed_aggregation_memory_efficient> + <parallel_view_processing>1</parallel_view_processing> + </default> + </profiles> + </yandex> + 03-database-ordinary.xml: | + <yandex> + <profiles> + <default> + <default_database_engine>Ordinary</default_database_engine> + </default> + </profiles> + </yandex> +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-etc-usersd-files +--- +apiVersion: v1 +data: null +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-keeper-etc-confd-files +--- +apiVersion: v1 +data: + 01-keeper-01-default-config.xml: | + <!-- IMPORTANT --> + <!-- This file is auto-generated --> + <!-- Do not edit this file - all changes would be lost --> + <!-- Edit appropriate template in the following folder: --> + <!-- deploy/builder/templates-config --> + <!-- IMPORTANT --> + <clickhouse> + <keeper_server> + <coordination_settings> + <min_session_timeout_ms>10000</min_session_timeout_ms> + <operation_timeout_ms>10000</operation_timeout_ms> + <raft_logs_level>information</raft_logs_level> + <session_timeout_ms>100000</session_timeout_ms> + </coordination_settings> + <hostname_checks_enabled>true</hostname_checks_enabled> + <log_storage_path>/var/lib/clickhouse-keeper/coordination/logs</log_storage_path> + <snapshot_storage_path>/var/lib/clickhouse-keeper/coordination/snapshots</snapshot_storage_path> + <storage_path>/var/lib/clickhouse-keeper</storage_path> + <tcp_port>2181</tcp_port> + </keeper_server> + <listen_host>::</listen_host> + <listen_host>0.0.0.0</listen_host> + <listen_try>1</listen_try> + <logger> + <console>1</console> + <level>information</level> + </logger> + <max_connections>4096</max_connections> + </clickhouse> + 01-keeper-02-readiness.xml: | + <!-- IMPORTANT --> + <!-- This file is auto-generated --> + <!-- Do not edit this file - all changes would be lost --> + <!-- Edit appropriate template in the following folder: --> + <!-- deploy/builder/templates-config --> + <!-- IMPORTANT --> + <clickhouse> + <keeper_server> + <http_control> + <port>9182</port> + <readiness> + <endpoint>/ready</endpoint> + </readiness> + </http_control> + </keeper_server> + </clickhouse> + 01-keeper-03-enable-reconfig.xml: |- + <!-- IMPORTANT --> + <!-- This file is auto-generated --> + <!-- Do not edit this file - all changes would be lost --> + <!-- Edit appropriate template in the following folder: --> + <!-- deploy/builder/templates-config --> + <!-- IMPORTANT --> + <clickhouse> + <keeper_server> + <enable_reconfiguration>false</enable_reconfiguration> + </keeper_server> + </clickhouse> +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-keeper-etc-configd-files +--- +apiVersion: v1 +data: + readme: Templates in this folder are packaged with an operator and available via 'useTemplate' +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-keeper-etc-templatesd-files +--- +apiVersion: v1 +data: null +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-keeper-etc-usersd-files diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/deployment.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/deployment.yaml new file mode 100644 index 0000000..a3ce2fe --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/deployment.yaml @@ -0,0 +1,208 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + template: + metadata: + annotations: + checksum/confd-files: 6a503b551565d380d3b4ea4b4c48a66629379558aa6cf0919d55f88f3cb4f56b + checksum/configd-files: a8b2e79da1b016cb2fa1e1b3d61f1edd8872af054a7419040c28b15a236bdc24 + checksum/files: 420251c3f150460f0b1b15211b9c0b965c16342055c2f3a5a57768d229b2ddd0 + checksum/keeper-confd-files: c17fcdc4ce46db171e44a8e5036cafb7173a9806470fafdef717cc3c01318470 + checksum/keeper-configd-files: beed860cfbb0c9e2a0b266f65dee8cbaf22990dc3e200c93b74e697b3cf4c9df + checksum/keeper-templatesd-files: a47abf45edf0f439048648e3d6d1ecede35e560a76f32d8a4aa6055e0fcf58a8 + checksum/keeper-usersd-files: 2fbf34a572990f10fe9bf11c9884c5e847a1a23a02f0728e83bac4f4f9036797 + checksum/templatesd-files: bfbe2e3230fe68164813fcaf74c5cc06b11878c26c764476399ac8803c8bde1c + checksum/usersd-files: c609c91ce94021945657aae781f199517cbc6f2c1f603238c88004f8acf84f95 + clickhouse-operator-metrics/port: "9999" + clickhouse-operator-metrics/scrape: "true" + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + spec: + containers: + - env: + - name: OPERATOR_POD_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OPERATOR_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPERATOR_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: OPERATOR_POD_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: OPERATOR_CONTAINER_CPU_REQUEST + valueFrom: + resourceFieldRef: + containerName: operator + resource: requests.cpu + - name: OPERATOR_CONTAINER_CPU_LIMIT + valueFrom: + resourceFieldRef: + containerName: operator + resource: limits.cpu + - name: OPERATOR_CONTAINER_MEM_REQUEST + valueFrom: + resourceFieldRef: + containerName: operator + resource: requests.memory + - name: OPERATOR_CONTAINER_MEM_LIMIT + valueFrom: + resourceFieldRef: + containerName: operator + resource: limits.memory + image: altinity/clickhouse-operator:0.25.3 + imagePullPolicy: IfNotPresent + name: operator + ports: + - containerPort: 9999 + name: op-metrics + resources: + requests: + cpu: 1000m + memory: 2Gi + securityContext: {} + volumeMounts: + - mountPath: /etc/clickhouse-operator + name: etc-clickhouse-operator-folder + - mountPath: /etc/clickhouse-operator/chi/conf.d + name: etc-clickhouse-operator-confd-folder + - mountPath: /etc/clickhouse-operator/chi/config.d + name: etc-clickhouse-operator-configd-folder + - mountPath: /etc/clickhouse-operator/chi/templates.d + name: etc-clickhouse-operator-templatesd-folder + - mountPath: /etc/clickhouse-operator/chi/users.d + name: etc-clickhouse-operator-usersd-folder + - mountPath: /etc/clickhouse-operator/chk/conf.d + name: etc-keeper-operator-confd-folder + - mountPath: /etc/clickhouse-operator/chk/keeper_config.d + name: etc-keeper-operator-configd-folder + - mountPath: /etc/clickhouse-operator/chk/templates.d + name: etc-keeper-operator-templatesd-folder + - mountPath: /etc/clickhouse-operator/chk/users.d + name: etc-keeper-operator-usersd-folder + - env: + - name: OPERATOR_POD_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OPERATOR_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPERATOR_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: OPERATOR_POD_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: OPERATOR_CONTAINER_CPU_REQUEST + valueFrom: + resourceFieldRef: + containerName: operator + resource: requests.cpu + - name: OPERATOR_CONTAINER_CPU_LIMIT + valueFrom: + resourceFieldRef: + containerName: operator + resource: limits.cpu + - name: OPERATOR_CONTAINER_MEM_REQUEST + valueFrom: + resourceFieldRef: + containerName: operator + resource: requests.memory + - name: OPERATOR_CONTAINER_MEM_LIMIT + valueFrom: + resourceFieldRef: + containerName: operator + resource: limits.memory + image: altinity/metrics-exporter:0.25.3 + imagePullPolicy: IfNotPresent + name: metrics-exporter + ports: + - containerPort: 8888 + name: ch-metrics + resources: + requests: + cpu: 100m + memory: 256Mi + securityContext: {} + volumeMounts: + - mountPath: /etc/clickhouse-operator + name: etc-clickhouse-operator-folder + - mountPath: /etc/clickhouse-operator/chi/conf.d + name: etc-clickhouse-operator-confd-folder + - mountPath: /etc/clickhouse-operator/chi/config.d + name: etc-clickhouse-operator-configd-folder + - mountPath: /etc/clickhouse-operator/chi/templates.d + name: etc-clickhouse-operator-templatesd-folder + - mountPath: /etc/clickhouse-operator/chi/users.d + name: etc-clickhouse-operator-usersd-folder + - mountPath: /etc/clickhouse-operator/chk/conf.d + name: etc-keeper-operator-confd-folder + - mountPath: /etc/clickhouse-operator/chk/keeper_config.d + name: etc-keeper-operator-configd-folder + - mountPath: /etc/clickhouse-operator/chk/templates.d + name: etc-keeper-operator-templatesd-folder + - mountPath: /etc/clickhouse-operator/chk/users.d + name: etc-keeper-operator-usersd-folder + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: signoz-telemetrystore-clickhouse-operator + volumes: + - configMap: + name: signoz-telemetrystore-clickhouse-operator-etc-files + name: etc-clickhouse-operator-folder + - configMap: + name: signoz-telemetrystore-clickhouse-operator-etc-confd-files + name: etc-clickhouse-operator-confd-folder + - configMap: + name: signoz-telemetrystore-clickhouse-operator-etc-configd-files + name: etc-clickhouse-operator-configd-folder + - configMap: + name: signoz-telemetrystore-clickhouse-operator-etc-templatesd-files + name: etc-clickhouse-operator-templatesd-folder + - configMap: + name: signoz-telemetrystore-clickhouse-operator-etc-usersd-files + name: etc-clickhouse-operator-usersd-folder + - configMap: + name: signoz-telemetrystore-clickhouse-operator-keeper-etc-confd-files + name: etc-keeper-operator-confd-folder + - configMap: + name: signoz-telemetrystore-clickhouse-operator-keeper-etc-configd-files + name: etc-keeper-operator-configd-folder + - configMap: + name: signoz-telemetrystore-clickhouse-operator-keeper-etc-templatesd-files + name: etc-keeper-operator-templatesd-folder + - configMap: + name: signoz-telemetrystore-clickhouse-operator-keeper-etc-usersd-files + name: etc-keeper-operator-usersd-folder diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/kustomization.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/kustomization.yaml new file mode 100644 index 0000000..6ed5d3e --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- serviceaccount.yaml +- configmap.yaml +- clusterrole.yaml +- clusterrolebinding.yaml +- service.yaml +- deployment.yaml diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/service.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/service.yaml new file mode 100644 index 0000000..98ebb4f --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-metrics +spec: + ports: + - name: ch-metrics + port: 8888 + - name: op-metrics + port: 9999 + selector: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/serviceaccount.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/serviceaccount.yaml new file mode 100644 index 0000000..6e3cb19 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/clickhouse-operator/serviceaccount.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/ingester/configmap.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/ingester/configmap.yaml new file mode 100644 index 0000000..88e9d4c --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/ingester/configmap.yaml @@ -0,0 +1,135 @@ +apiVersion: v1 +data: + ingester.yaml: |- + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 1s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 1s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://signoz-clickhouse:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://signoz-clickhouse:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://signoz-clickhouse:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://signoz-clickhouse:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: 'server_endpoint: ws://signoz-signoz:4320/v1/opamp' +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-ingester + name: signoz-ingester diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/ingester/deployment.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/ingester/deployment.yaml new file mode 100644 index 0000000..c051469 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/ingester/deployment.yaml @@ -0,0 +1,121 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-ingester + name: signoz-ingester +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-ingester + template: + metadata: + labels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-ingester + spec: + containers: + - args: + - --config=/conf/ingester.yaml + - --manager-config=/conf/opamp.yaml + - --copy-path=/var/tmp/collector-config.yaml + command: + - /signoz-otel-collector + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: signoz/signoz-otel-collector:latest + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 6 + httpGet: + path: / + port: 13133 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: collector + ports: + - containerPort: 4317 + name: otlp-grpc + protocol: TCP + - containerPort: 4318 + name: otlp-http + protocol: TCP + - containerPort: 13133 + name: health + protocol: TCP + readinessProbe: + failureThreshold: 6 + httpGet: + path: / + port: 13133 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + resources: + limits: + cpu: "2" + memory: 4Gi + requests: + cpu: "1" + memory: 2Gi + volumeMounts: + - mountPath: /conf + name: ingester-config-vol + initContainers: + - args: + - migrate + - sync + - check + command: + - /signoz-otel-collector + env: + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_CLUSTER + value: cluster + - name: SIGNOZ_OTEL_COLLECTOR_TIMEOUT + value: 10m + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_REPLICATION + value: "true" + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN + value: tcp://signoz-clickhouse:9000 + image: signoz/signoz-otel-collector:latest + imagePullPolicy: IfNotPresent + name: check + serviceAccountName: signoz-ingester + volumes: + - configMap: + name: signoz-ingester + name: ingester-config-vol diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/ingester/kustomization.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/ingester/kustomization.yaml new file mode 100644 index 0000000..025ce64 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/ingester/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- serviceaccount.yaml +- configmap.yaml +- service.yaml +- deployment.yaml diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/ingester/service.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/ingester/service.yaml new file mode 100644 index 0000000..d26ad2b --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/ingester/service.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-ingester + name: signoz-ingester +spec: + ports: + - name: otlp-grpc + port: 4317 + protocol: TCP + targetPort: 4317 + - name: otlp-http + port: 4318 + protocol: TCP + targetPort: 4318 + - name: health + port: 13133 + protocol: TCP + targetPort: 13133 + selector: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-ingester + type: ClusterIP diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/ingester/serviceaccount.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/ingester/serviceaccount.yaml new file mode 100644 index 0000000..472df92 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/ingester/serviceaccount.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-ingester + name: signoz-ingester diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/kustomization.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/kustomization.yaml new file mode 100644 index 0000000..2f960b3 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: signoz +resources: +- namespace.yaml +- clickhouse-operator +- telemetrystore/clickhouse +- telemetrykeeper/clickhousekeeper +- telemetrystore-migrator +- metastore/postgres +- signoz +- ingester diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/metastore/postgres/kustomization.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/metastore/postgres/kustomization.yaml new file mode 100644 index 0000000..7fcd38a --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/metastore/postgres/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- serviceaccount.yaml +- service.yaml +- statefulset.yaml diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/metastore/postgres/service.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/metastore/postgres/service.yaml new file mode 100644 index 0000000..b858a59 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/metastore/postgres/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: metastore + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-metastore + name: signoz-metastore +spec: + ports: + - name: tcp-postgresql + port: 5432 + protocol: TCP + targetPort: tcp-postgresql + selector: + app.kubernetes.io/component: metastore + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-metastore + type: ClusterIP diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/metastore/postgres/serviceaccount.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/metastore/postgres/serviceaccount.yaml new file mode 100644 index 0000000..f64d89d --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/metastore/postgres/serviceaccount.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: metastore + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-metastore + name: signoz-metastore diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/metastore/postgres/statefulset.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/metastore/postgres/statefulset.yaml new file mode 100644 index 0000000..61be1e1 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/metastore/postgres/statefulset.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/component: metastore + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-metastore + name: signoz-metastore +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: metastore + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-metastore + serviceName: signoz-metastore + template: + metadata: + labels: + app.kubernetes.io/component: metastore + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-metastore + spec: + containers: + - env: + - name: POSTGRESQL_VOLUME_DIR + value: /var/lib/postgresql + - name: POSTGRESQL_PORT_NUMBER + value: "5432" + - name: PGDATA + value: /var/lib/postgresql/data + - name: POSTGRES_DB + value: signoz + - name: POSTGRES_PASSWORD + value: signoz + - name: POSTGRES_USER + value: signoz + image: postgres:16 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - /bin/sh + - -c + - exec pg_isready -U "$POSTGRES_USER" -h 127.0.0.1 -p 5432 + failureThreshold: 6 + initialDelaySeconds: 20 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: postgres + ports: + - containerPort: 5432 + name: tcp-postgresql + protocol: TCP + readinessProbe: + exec: + command: + - /bin/sh + - -c + - exec pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" -h 127.0.0.1 -p + 5432 + failureThreshold: 6 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 3 + resources: + requests: + cpu: 100m + memory: 128Mi + volumeMounts: + - mountPath: /var/lib/postgresql + name: pgdata + serviceAccountName: signoz-metastore + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: pgdata + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + storageClassName: gp3 diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/namespace.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/namespace.yaml new file mode 100644 index 0000000..b0391bc --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: signoz diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/signoz/kustomization.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/signoz/kustomization.yaml new file mode 100644 index 0000000..7fcd38a --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/signoz/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- serviceaccount.yaml +- service.yaml +- statefulset.yaml diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/signoz/service.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/signoz/service.yaml new file mode 100644 index 0000000..6759417 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/signoz/service.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: nlb + labels: + app.kubernetes.io/component: signoz + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-signoz + name: signoz-signoz +spec: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: http + - name: opamp + port: 4320 + protocol: TCP + targetPort: opamp + selector: + app.kubernetes.io/component: signoz + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-signoz + type: LoadBalancer diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/signoz/serviceaccount.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/signoz/serviceaccount.yaml new file mode 100644 index 0000000..390b499 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/signoz/serviceaccount.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: signoz + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-signoz + name: signoz-signoz diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/signoz/statefulset.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/signoz/statefulset.yaml new file mode 100644 index 0000000..0eb3c19 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/signoz/statefulset.yaml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/component: signoz + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-signoz + name: signoz-signoz +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: signoz + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-signoz + serviceName: signoz-signoz + template: + metadata: + labels: + app.kubernetes.io/component: signoz + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-signoz + spec: + containers: + - env: + - name: SIGNOZ_SQLSTORE_POSTGRES_DSN + value: postgres://signoz:signoz@signoz-metastore:5432/signoz?sslmode=disable + - name: SIGNOZ_SQLSTORE_PROVIDER + value: postgres + - name: SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN + value: tcp://signoz-clickhouse:9000 + - name: SIGNOZ_TELEMETRYSTORE_PROVIDER + value: clickhouse + image: signoz/signoz:latest + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 6 + httpGet: + path: /api/v1/health + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: signoz + ports: + - containerPort: 8080 + name: http + protocol: TCP + - containerPort: 4320 + name: opamp + protocol: TCP + readinessProbe: + failureThreshold: 6 + httpGet: + path: /api/v1/health + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: 500m + memory: 1Gi + nodeSelector: + node-role.kubernetes.io/observability: "" + serviceAccountName: signoz-signoz + tolerations: + - effect: NoSchedule + key: dedicated + operator: Equal + value: signoz diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrykeeper/clickhousekeeper/clickhousekeeperinstallation.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrykeeper/clickhousekeeper/clickhousekeeperinstallation.yaml new file mode 100644 index 0000000..7b1be82 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrykeeper/clickhousekeeper/clickhousekeeperinstallation.yaml @@ -0,0 +1,49 @@ +apiVersion: clickhouse-keeper.altinity.com/v1 +kind: ClickHouseKeeperInstallation +metadata: + name: signoz-clickhouse-keeper +spec: + configuration: + clusters: + - layout: + replicasCount: 1 + name: cluster + settings: + keeper_server/coordination_settings/raft_logs_level: information + keeper_server/four_letter_word_white_list: '*' + listen_host: 0.0.0.0 + logger/console: "true" + logger/level: trace + defaults: + templates: + dataVolumeClaimTemplate: default + podTemplate: default + templates: + podTemplates: + - metadata: + labels: + app.kubernetes.io/component: telemetrykeeper + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-clickhouse-keeper + app.kubernetes.io/part-of: signoz + name: default + spec: + containers: + - image: clickhouse/clickhouse-keeper:25.5.6 + imagePullPolicy: IfNotPresent + name: clickhouse-keeper + resources: + requests: + cpu: 1 + memory: 1Gi + securityContext: + fsGroup: 101 + volumeClaimTemplates: + - name: default + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrykeeper/clickhousekeeper/kustomization.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrykeeper/clickhousekeeper/kustomization.yaml new file mode 100644 index 0000000..e2d3083 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrykeeper/clickhousekeeper/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- clickhousekeeperinstallation.yaml diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrystore-migrator/job.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrystore-migrator/job.yaml new file mode 100644 index 0000000..b0b5205 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrystore-migrator/job.yaml @@ -0,0 +1,99 @@ +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/component: schema-migrator + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-telemetrystore-migrator + name: signoz-telemetrystore-migrator +spec: + backoffLimit: 6 + template: + metadata: + labels: + app.kubernetes.io/component: schema-migrator + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-telemetrystore-migrator + spec: + containers: + - args: + - migrate + - async + - up + env: + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_CLUSTER + value: cluster + - name: SIGNOZ_OTEL_COLLECTOR_TIMEOUT + value: 10m + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_REPLICATION + value: "true" + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN + value: tcp://signoz-clickhouse:9000 + image: signoz/signoz-otel-collector:latest + imagePullPolicy: IfNotPresent + name: async + resources: + requests: + cpu: 10m + memory: 16Mi + initContainers: + - args: + - migrate + - ready + env: + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_CLUSTER + value: cluster + - name: SIGNOZ_OTEL_COLLECTOR_TIMEOUT + value: 10m + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_REPLICATION + value: "true" + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN + value: tcp://signoz-clickhouse:9000 + image: signoz/signoz-otel-collector:latest + imagePullPolicy: IfNotPresent + name: ready + resources: + requests: + cpu: 10m + memory: 16Mi + - args: + - migrate + - bootstrap + env: + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_CLUSTER + value: cluster + - name: SIGNOZ_OTEL_COLLECTOR_TIMEOUT + value: 10m + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_REPLICATION + value: "true" + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN + value: tcp://signoz-clickhouse:9000 + image: signoz/signoz-otel-collector:latest + imagePullPolicy: IfNotPresent + name: bootstrap + resources: + requests: + cpu: 10m + memory: 16Mi + - args: + - migrate + - sync + - up + env: + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_CLUSTER + value: cluster + - name: SIGNOZ_OTEL_COLLECTOR_TIMEOUT + value: 10m + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_REPLICATION + value: "true" + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN + value: tcp://signoz-clickhouse:9000 + image: signoz/signoz-otel-collector:latest + imagePullPolicy: IfNotPresent + name: sync + resources: + requests: + cpu: 10m + memory: 16Mi + restartPolicy: OnFailure diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrystore-migrator/kustomization.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrystore-migrator/kustomization.yaml new file mode 100644 index 0000000..ac2f8e0 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrystore-migrator/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- job.yaml diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrystore/clickhouse/clickhouseinstallation.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrystore/clickhouse/clickhouseinstallation.yaml new file mode 100644 index 0000000..46a7d6a --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrystore/clickhouse/clickhouseinstallation.yaml @@ -0,0 +1,279 @@ +apiVersion: clickhouse.altinity.com/v1 +kind: ClickHouseInstallation +metadata: + labels: + app.kubernetes.io/component: telemetrystore + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-clickhouse + app.kubernetes.io/part-of: signoz + app.kubernetes.io/version: 25.5.6 + name: signoz-clickhouse +spec: + configuration: + clusters: + - layout: + replicasCount: 0 + shardsCount: 1 + name: cluster + files: + config.d/crash.yaml: | + send_crash_reports: + enabled: false + config.d/extra.yaml: | + dictionaries_config: '*_dictionary.xml' + format_schema_path: /etc/clickhouse-server/config.d/ + prometheus: + endpoint: /metrics + port: 9363 + tmp_path: /var/lib/clickhouse/tmp/ + user_defined_executable_functions_config: /etc/clickhouse-server/functions/custom-functions.xml + user_directories: + users_xml: + path: users.xml + user_files_path: /var/lib/clickhouse/user_files/ + user_scripts_path: /var/lib/clickhouse/user_scripts/ + config.d/formatting.yaml: | + logger: + console: 1 + count: 10 + formatting: + names: + date_time: date_time + level: level + logger_name: logger_name + message: message + query_id: query_id + source_file: source_file + source_line: source_line + thread_id: thread_id + thread_name: thread_name + type: json + level: information + size: 1000M + config.d/load.yaml: | + async_load_databases: false + config.d/path.yaml: | + merge_tree: + storage_policy: data + path: /var/lib/clickhouse/ + config.d/storage.yaml: | + storage_configuration: + disks: + data0: + keep_free_space_bytes: 10485760 + path: /mnt/data-0/ + default: {} + policies: + data: + volumes: + data0: + disk: data0 + default: + volumes: + default: + disk: default + config.d/system_log.yaml: | + error_log: + ttl: event_date + INTERVAL 7 DAY DELETE + latency_log: + ttl: event_date + INTERVAL 3 DAY DELETE + query_metric_log: + ttl: event_date + INTERVAL 3 DAY DELETE + text_log: + ttl: event_date + INTERVAL 3 DAY DELETE + events.proto: | + syntax = "proto3"; + message Event { + string uuid = 1; + string event = 2; + string properties = 3; + string timestamp = 4; + uint64 team_id = 5; + string distinct_id = 6; + string created_at = 7; + string elements_chain = 8; + } + profiles: + admin/max_threads: 16 + admin/query_plan_max_limit_for_lazy_materialization: "0" + admin/secondary_indices_enable_bulk_filtering: "0" + default/allow_experimental_window_functions: "1" + default/allow_nondeterministic_mutations: "1" + default/load_balancing: random + default/log_queries: 1 + default/max_memory_usage: 10000000000 + default/max_threads: 16 + default/query_plan_max_limit_for_lazy_materialization: "0" + default/secondary_indices_enable_bulk_filtering: "0" + quotas: + default/interval/duration: 3600 + default/interval/errors: 0 + default/interval/execution_time: 0 + default/interval/queries: 0 + default/interval/read_rows: 0 + default/interval/result_rows: 0 + users: + admin/networks/ip: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 172.16.0.0/12 + - 192.0.0.0/24 + - 198.18.0.0/15 + - 192.168.0.0/16 + admin/password: 27ff0399-0d3a-4bd8-919d-17c2181e6fb9 + admin/profile: default + admin/quota: default + default/access_management: 1 + default/named_collection_control: 1 + default/networks/ip: ::/0 + default/password: "" + default/profile: default + default/quota: default + default/show_named_collection: 1 + default/show_named_collection_secrets: 1 + zookeeper: + nodes: + - host: keeper-signoz-clickhouse-keeper + port: 2181 + defaults: + templates: + podTemplate: default + serviceTemplate: default + templates: + podTemplates: + - metadata: + labels: + app.kubernetes.io/component: telemetrystore + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-clickhouse + app.kubernetes.io/version: 25.5.6 + name: default + spec: + containers: + - command: + - /bin/bash + - -c + - /usr/bin/clickhouse-server --config-file=/etc/clickhouse-server/config.xml + image: clickhouse/clickhouse-server:25.5.6 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 10 + httpGet: + path: /ping + port: http + initialDelaySeconds: 60 + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 1 + name: clickhouse + ports: + - containerPort: 8123 + name: http + - containerPort: 9000 + name: client + - containerPort: 9009 + name: interserver + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ping + port: http + initialDelaySeconds: 10 + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: "4" + memory: 8Gi + requests: + cpu: "2" + memory: 4Gi + startupProbe: + failureThreshold: 30 + httpGet: + path: /ping + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /var/lib/clickhouse/user_scripts + name: shared-binary-volume + - mountPath: /etc/clickhouse-server/functions + name: custom-functions-volume + - mountPath: /var/lib/clickhouse + name: default + - mountPath: /mnt/data-0 + name: data-0 + initContainers: + - command: + - sh + - -c + - | + set -e + version="v0.0.1" + node_os=$(uname -s | tr '[:upper:]' '[:lower:]') + node_arch=$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) + echo "Fetching histogram-binary for ${node_os}/${node_arch}" + cd /tmp + wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F${version}/histogram-quantile_${node_os}_${node_arch}.tar.gz" + tar -xzf histogram-quantile.tar.gz + chmod +x histogram-quantile + mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile + echo "histogram-quantile installed successfully" + image: docker.io/alpine:3.18.2 + imagePullPolicy: IfNotPresent + name: histogram + volumeMounts: + - mountPath: /var/lib/clickhouse/user_scripts + name: shared-binary-volume + securityContext: + fsGroup: 101 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 101 + runAsUser: 101 + volumes: + - emptyDir: {} + name: shared-binary-volume + - configMap: + name: signoz-clickhouse-custom-functions + name: custom-functions-volume + serviceTemplates: + - generateName: signoz-clickhouse + metadata: + labels: + app.kubernetes.io/component: telemetrystore + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-clickhouse + app.kubernetes.io/part-of: signoz + name: default + spec: + ports: + - name: http + port: 8123 + - name: tcp + port: 9000 + type: ClusterIP + volumeClaimTemplates: + - name: data-0 + reclaimPolicy: Retain + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi + storageClassName: gp3 + - name: default + reclaimPolicy: Retain + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: gp3 diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrystore/clickhouse/configmap.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrystore/clickhouse/configmap.yaml new file mode 100644 index 0000000..ec8f548 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrystore/clickhouse/configmap.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +data: + custom-functions.xml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: telemetrystore + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-clickhouse + app.kubernetes.io/part-of: signoz + name: signoz-clickhouse-custom-functions diff --git a/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrystore/clickhouse/kustomization.yaml b/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrystore/clickhouse/kustomization.yaml new file mode 100644 index 0000000..a6835a5 --- /dev/null +++ b/docs/examples/kubernetes/kustomize-patches/pours/deployment/telemetrystore/clickhouse/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- configmap.yaml +- clickhouseinstallation.yaml diff --git a/docs/examples/kubernetes/kustomize/README.md b/docs/examples/kubernetes/kustomize/README.md new file mode 100644 index 0000000..7a4e20b --- /dev/null +++ b/docs/examples/kubernetes/kustomize/README.md @@ -0,0 +1,178 @@ +# Kubernetes with Kustomize + +| Field | Value | +| --- | --- | +| **Mode** | `kubernetes` | +| **Flavor** | `kustomize` | +| **Platform** | `-` | + +## Overview + +Deploys SigNoz on Kubernetes using Kustomize. Foundry generates per-component directories with Kubernetes manifests and a root `kustomization.yaml`. + +## Prerequisites + +- Kubernetes cluster (1.24+) +- `kubectl` with kustomize support + +## Configuration + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + flavor: kustomize + mode: kubernetes +``` + +## Deploy + +```bash +foundryctl cast -f casting.yaml +``` + +Or step by step: + +```bash +# Generate manifests +foundryctl forge -f casting.yaml + +# Apply CRDs first (cast does this automatically) +kubectl apply -f https://raw.githubusercontent.com/Altinity/clickhouse-operator/0.25.3/deploy/operatorhub/0.25.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +kubectl apply -f https://raw.githubusercontent.com/Altinity/clickhouse-operator/0.25.3/deploy/operatorhub/0.25.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +kubectl apply -f https://raw.githubusercontent.com/Altinity/clickhouse-operator/0.25.3/deploy/operatorhub/0.25.3/clickhouseoperatorconfigurations.clickhouse.altinity.com.crd.yaml +kubectl apply -f https://raw.githubusercontent.com/Altinity/clickhouse-operator/0.25.3/deploy/operatorhub/0.25.3/clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml + +# Apply with kubectl +kubectl apply -k pours/deployment/ +``` + +> [!NOTE] +> `foundryctl cast` automatically fetches and applies the four Altinity ClickHouse Operator CRDs (v0.25.3) from GitHub before running `kubectl apply -k`. If you apply manually, you must install the CRDs first or the ClickHouseInstallation and ClickHouseKeeperInstallation resources will fail to create. + +## Generated output + +```text +pours/deployment/ + kustomization.yaml + namespace.yaml + signoz/ + statefulset.yaml + service.yaml + serviceaccount.yaml + kustomization.yaml + ingester/ + deployment.yaml + service.yaml + configmap.yaml + serviceaccount.yaml + kustomization.yaml + telemetrystore/ + clickhouse/ + clickhouseinstallation.yaml + configmap.yaml + kustomization.yaml + clickhouse-operator/ + deployment.yaml + clusterrole.yaml + clusterrolebinding.yaml + configmap.yaml + service.yaml + serviceaccount.yaml + kustomization.yaml + telemetrykeeper/ + clickhousekeeper/ + clickhousekeeperinstallation.yaml + kustomization.yaml + metastore/ + postgres/ + statefulset.yaml + service.yaml + serviceaccount.yaml + kustomization.yaml + telemetrystore-migrator/ + job.yaml + kustomization.yaml +``` + +## After deployment + +```bash +# Check pod status +kubectl get pods -n signoz + +# Port-forward the SigNoz UI +kubectl port-forward svc/signoz -n signoz 8080:8080 +``` + +Open `http://localhost:8080` to access the SigNoz UI. + +## Customization + +To set resource limits, storage classes, or scheduling constraints on the generated manifests, use [patches](../../../concepts/patches.md). See the [kustomize-patches](../kustomize-patches/) example for a complete working configuration. + +### Native Kustomize patches + +Since Foundry generates standard Kustomize bases, you can also use native Kustomize patches on the generated `kustomization.yaml`. This lets you use strategic merge patches or overlays for environment-specific customization without re-forging. + +Use a Foundry patch to inject a `patches` block into the root `kustomization.yaml`: + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + flavor: kustomize + mode: kubernetes + patches: + - target: "deployment/kustomization.yaml" + operations: + - op: add + path: /patches + value: + - target: + kind: StatefulSet + name: signoz-signoz + patch: |- + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: signoz-signoz + spec: + template: + spec: + nodeSelector: + node-role.kubernetes.io/observability: "" +``` + +Or create an overlay directory that references the generated base: + +``` +my-deployment/ +├── base/ # Copy of pours/deployment/ +│ └── ... +└── overlays/ + └── prod/ + ├── kustomization.yaml + └── increase-resources.yaml +``` + +```yaml +# overlays/prod/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../base +patches: +- path: increase-resources.yaml + target: + kind: StatefulSet + name: signoz-clickhouse +``` + +```bash +kubectl apply -k overlays/prod/ +``` diff --git a/docs/examples/kubernetes/kustomize/casting.yaml b/docs/examples/kubernetes/kustomize/casting.yaml new file mode 100644 index 0000000..33c1819 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/casting.yaml @@ -0,0 +1,8 @@ +apiVersion: v1alpha1 +kind: Installation +metadata: + name: signoz +spec: + deployment: + flavor: kustomize + mode: kubernetes diff --git a/docs/examples/kubernetes/kustomize/casting.yaml.lock b/docs/examples/kubernetes/kustomize/casting.yaml.lock new file mode 100644 index 0000000..dfddb94 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/casting.yaml.lock @@ -0,0 +1,825 @@ +apiVersion: v1alpha1 +kind: Installation +metadata: + name: signoz +spec: + deployment: + flavor: kustomize + mode: kubernetes + ingester: + spec: + cluster: + replicas: 1 + config: + data: + ingester.yaml: | + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 5s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://signoz-clickhouse:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://signoz-clickhouse:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://signoz-clickhouse:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://signoz-clickhouse:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + metadataexporter: + enabled: true + dsn: tcp://signoz-clickhouse:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + - metadataexporter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + - metadataexporter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + - metadataexporter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: | + server_endpoint: ws://signoz-signoz:4320/v1/opamp + enabled: true + env: + SIGNOZ_OTEL_COLLECTOR_TIMEOUT: 10m + image: signoz/signoz-otel-collector:latest + version: latest + status: + addresses: + otlp: null + config: + data: + ingester.yaml: | + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 5s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://signoz-clickhouse:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://signoz-clickhouse:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://signoz-clickhouse:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://signoz-clickhouse:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + metadataexporter: + enabled: true + dsn: tcp://signoz-clickhouse:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + - metadataexporter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + - metadataexporter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + - metadataexporter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: | + server_endpoint: ws://signoz-signoz:4320/v1/opamp + env: + SIGNOZ_OTEL_COLLECTOR_TIMEOUT: 10m + metastore: + kind: postgres + spec: + cluster: + replicas: 1 + config: {} + enabled: true + env: + POSTGRES_DB: signoz + POSTGRES_PASSWORD: signoz + POSTGRES_USER: signoz + image: postgres:16 + version: "16" + status: + addresses: + dsn: + - postgres://signoz-metastore:5432 + config: {} + env: + POSTGRES_DB: signoz + POSTGRES_PASSWORD: signoz + POSTGRES_USER: signoz + signoz: + spec: + cluster: + replicas: 1 + config: {} + enabled: true + env: + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@signoz-metastore:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://signoz-clickhouse:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + image: signoz/signoz:latest + version: latest + status: + addresses: + apiserver: null + opamp: + - ws://signoz-signoz:4320 + config: {} + env: + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@signoz-metastore:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://signoz-clickhouse:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + telemetrykeeper: + kind: clickhousekeeper + spec: + cluster: + replicas: 1 + config: + data: + keeper-0.yaml: | + listen_host: 0.0.0.0 + logger: + level: information + console: true + keeper_server: + four_letter_word_white_list: "*" + coordination_settings: + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: signoz-clickhouse-keeper-0 + port: 9234 + id: 0 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 + enabled: true + image: clickhouse/clickhouse-keeper:25.5.6 + version: 25.5.6 + status: + addresses: + client: + - tcp://signoz-clickhouse-keeper-0:9181 + raft: + - tcp://signoz-clickhouse-keeper-0:9234 + config: + data: + keeper-0.yaml: | + listen_host: 0.0.0.0 + logger: + level: information + console: true + keeper_server: + four_letter_word_white_list: "*" + coordination_settings: + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: signoz-clickhouse-keeper-0 + port: 9234 + id: 0 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 + telemetrystore: + kind: clickhouse + spec: + cluster: + replicas: 0 + shards: 1 + config: + data: + config-0-0.yaml: | + async_load_databases: false + asynchronous_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE + dictionaries_config: '*_dictionary.xml' + display_name: cluster + distributed_ddl: + path: /clickhouse/task_queue/ddl + error_log: + ttl: event_date + INTERVAL 7 DAY DELETE + format_schema_path: /etc/clickhouse-server/config.d/ + http_port: 8123 + interserver_http_port: 9009 + latency_log: + ttl: event_date + INTERVAL 3 DAY DELETE + listen_host: 0.0.0.0 + logger: + console: 1 + count: 10 + formatting: + names: + date_time: date_time + level: level + logger_name: logger_name + message: message + query_id: query_id + source_file: source_file + source_line: source_line + thread_id: thread_id + thread_name: thread_name + type: json + level: information + size: 1000M + macros: + replica: "00" + shard: "00" + merge_tree: + storage_policy: data + metric_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE + part_log: + ttl: event_date + INTERVAL 1 DAY DELETE + path: /var/lib/clickhouse/ + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE + profiles: + admin: + max_threads: 16 + query_plan_max_limit_for_lazy_materialization: "0" + secondary_indices_enable_bulk_filtering: "0" + default: + allow_experimental_window_functions: "1" + allow_nondeterministic_mutations: "1" + allow_simdjson: 0 + load_balancing: random + log_queries: 1 + max_threads: 16 + query_plan_max_limit_for_lazy_materialization: "0" + secondary_indices_enable_bulk_filtering: "0" + prometheus: + endpoint: /metrics + port: 9363 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: event_date + INTERVAL 1 DAY DELETE + query_metric_log: + ttl: event_date + INTERVAL 3 DAY DELETE + query_thread_log: + ttl: event_date + INTERVAL 1 DAY DELETE + query_views_log: + ttl: event_date + INTERVAL 1 DAY DELETE + quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 + remote_servers: + cluster: + shard: + - replica: + - host: signoz-clickhouse + port: 9000 + send_crash_reports: + enabled: false + session_log: + ttl: event_date + INTERVAL 1 DAY DELETE + storage_configuration: + disks: + data0: + keep_free_space_bytes: 10485760 + path: /mnt/data-0/ + default: {} + policies: + data: + volumes: + data0: + disk: data0 + default: + volumes: + default: + disk: default + tcp_port: 9000 + text_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 3 DAY DELETE + tmp_path: /var/lib/clickhouse/tmp/ + trace_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE + user_defined_executable_functions_config: /etc/clickhouse-server/functions/custom-functions.xml + user_directories: + users_xml: + path: users.xml + user_files_path: /var/lib/clickhouse/user_files/ + user_scripts_path: /var/lib/clickhouse/user_scripts/ + users: + admin: + networks: + ip: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 172.16.0.0/12 + - 192.0.0.0/24 + - 198.18.0.0/15 + - 192.168.0.0/16 + password: 27ff0399-0d3a-4bd8-919d-17c2181e6fb9 + profile: default + quota: default + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 + zookeeper: + node: + - host: signoz-clickhouse-keeper-0 + port: 9181 + zookeeper_log: + ttl: event_date + INTERVAL 1 DAY DELETE + functions.yaml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable + enabled: true + image: clickhouse/clickhouse-server:25.5.6 + version: 25.5.6 + status: + addresses: + tcp: + - tcp://signoz-clickhouse:9000 + config: + data: + config-0-0.yaml: | + async_load_databases: false + asynchronous_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE + dictionaries_config: '*_dictionary.xml' + display_name: cluster + distributed_ddl: + path: /clickhouse/task_queue/ddl + error_log: + ttl: event_date + INTERVAL 7 DAY DELETE + format_schema_path: /etc/clickhouse-server/config.d/ + http_port: 8123 + interserver_http_port: 9009 + latency_log: + ttl: event_date + INTERVAL 3 DAY DELETE + listen_host: 0.0.0.0 + logger: + console: 1 + count: 10 + formatting: + names: + date_time: date_time + level: level + logger_name: logger_name + message: message + query_id: query_id + source_file: source_file + source_line: source_line + thread_id: thread_id + thread_name: thread_name + type: json + level: information + size: 1000M + macros: + replica: "00" + shard: "00" + merge_tree: + storage_policy: data + metric_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE + part_log: + ttl: event_date + INTERVAL 1 DAY DELETE + path: /var/lib/clickhouse/ + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE + profiles: + admin: + max_threads: 16 + query_plan_max_limit_for_lazy_materialization: "0" + secondary_indices_enable_bulk_filtering: "0" + default: + allow_experimental_window_functions: "1" + allow_nondeterministic_mutations: "1" + allow_simdjson: 0 + load_balancing: random + log_queries: 1 + max_threads: 16 + query_plan_max_limit_for_lazy_materialization: "0" + secondary_indices_enable_bulk_filtering: "0" + prometheus: + endpoint: /metrics + port: 9363 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: event_date + INTERVAL 1 DAY DELETE + query_metric_log: + ttl: event_date + INTERVAL 3 DAY DELETE + query_thread_log: + ttl: event_date + INTERVAL 1 DAY DELETE + query_views_log: + ttl: event_date + INTERVAL 1 DAY DELETE + quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 + remote_servers: + cluster: + shard: + - replica: + - host: signoz-clickhouse + port: 9000 + send_crash_reports: + enabled: false + session_log: + ttl: event_date + INTERVAL 1 DAY DELETE + storage_configuration: + disks: + data0: + keep_free_space_bytes: 10485760 + path: /mnt/data-0/ + default: {} + policies: + data: + volumes: + data0: + disk: data0 + default: + volumes: + default: + disk: default + tcp_port: 9000 + text_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 3 DAY DELETE + tmp_path: /var/lib/clickhouse/tmp/ + trace_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE + user_defined_executable_functions_config: /etc/clickhouse-server/functions/custom-functions.xml + user_directories: + users_xml: + path: users.xml + user_files_path: /var/lib/clickhouse/user_files/ + user_scripts_path: /var/lib/clickhouse/user_scripts/ + users: + admin: + networks: + ip: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 172.16.0.0/12 + - 192.0.0.0/24 + - 198.18.0.0/15 + - 192.168.0.0/16 + password: 27ff0399-0d3a-4bd8-919d-17c2181e6fb9 + profile: default + quota: default + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 + zookeeper: + node: + - host: signoz-clickhouse-keeper-0 + port: 9181 + zookeeper_log: + ttl: event_date + INTERVAL 1 DAY DELETE + functions.yaml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable + extras: + _overrides: | + async_load_databases: false + error_log: + ttl: event_date + INTERVAL 7 DAY DELETE + format_schema_path: /etc/clickhouse-server/config.d/ + latency_log: + ttl: event_date + INTERVAL 3 DAY DELETE + logger: + formatting: + names: + date_time: date_time + level: level + logger_name: logger_name + message: message + query_id: query_id + source_file: source_file + source_line: source_line + thread_id: thread_id + thread_name: thread_name + type: json + merge_tree: + storage_policy: data + profiles: + admin: + max_threads: 16 + query_plan_max_limit_for_lazy_materialization: "0" + secondary_indices_enable_bulk_filtering: "0" + default: + allow_experimental_window_functions: "1" + allow_nondeterministic_mutations: "1" + max_threads: 16 + query_plan_max_limit_for_lazy_materialization: "0" + secondary_indices_enable_bulk_filtering: "0" + prometheus: + endpoint: /metrics + port: 9363 + query_metric_log: + ttl: event_date + INTERVAL 3 DAY DELETE + send_crash_reports: + enabled: false + storage_configuration: + disks: + data0: + keep_free_space_bytes: 10485760 + path: /mnt/data-0/ + default: {} + policies: + data: + volumes: + data0: + disk: data0 + default: + volumes: + default: + disk: default + text_log: + ttl: event_date + INTERVAL 3 DAY DELETE + user_defined_executable_functions_config: /etc/clickhouse-server/functions/custom-functions.xml + users: + admin: + networks: + ip: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 172.16.0.0/12 + - 192.0.0.0/24 + - 198.18.0.0/15 + - 192.168.0.0/16 + password: 27ff0399-0d3a-4bd8-919d-17c2181e6fb9 + profile: default + quota: default diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/clusterrole.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/clusterrole.yaml new file mode 100644 index 0000000..4cbeaa5 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/clusterrole.yaml @@ -0,0 +1,184 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator +rules: +- apiGroups: + - "" + resources: + - configmaps + - services + - persistentvolumeclaims + - secrets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch + - delete +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete +- apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - patch + - update + - delete +- apiGroups: + - apps + resourceNames: + - clickhouse-operator + resources: + - deployments + verbs: + - get + - patch + - update + - delete +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list +- apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations + verbs: + - get + - list + - watch + - patch + - update + - delete +- apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallationtemplates + - clickhouseoperatorconfigurations + verbs: + - get + - list + - watch +- apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/finalizers + - clickhouseinstallationtemplates/finalizers + - clickhouseoperatorconfigurations/finalizers + verbs: + - update +- apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/status + - clickhouseinstallationtemplates/status + - clickhouseoperatorconfigurations/status + verbs: + - get + - update + - patch + - create + - delete +- apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations + verbs: + - get + - list + - watch + - patch + - update + - delete +- apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/finalizers + verbs: + - update +- apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/status + verbs: + - get + - update + - patch + - create + - delete diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/clusterrolebinding.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/clusterrolebinding.yaml new file mode 100644 index 0000000..baf43b7 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: signoz-telemetrystore-clickhouse-operator +subjects: +- kind: ServiceAccount + name: signoz-telemetrystore-clickhouse-operator + namespace: signoz diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/configmap.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/configmap.yaml new file mode 100644 index 0000000..ec74edf --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/configmap.yaml @@ -0,0 +1,576 @@ +apiVersion: v1 +data: null +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-etc-confd-files +--- +apiVersion: v1 +data: + 01-clickhouse-01-listen.xml: | + <clickhouse> + <!-- Listen wildcard address to allow accepting connections from other containers and host network. --> + <listen_host>::</listen_host> + <listen_host>0.0.0.0</listen_host> + <listen_try>1</listen_try> + </clickhouse> + 01-clickhouse-02-logger.xml: | + <clickhouse> + <logger> + <!-- Possible levels: https://github.com/pocoproject/poco/blob/devel/Foundation/include/Poco/Logger.h#L105 --> + <level>warning</level> + <formatting> + <type>json</type> + </formatting> + <log>/var/log/clickhouse-server/clickhouse-server.log</log> + <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog> + <size>1000M</size> + <count>10</count> + <!-- Default behavior is autodetection (log to console if not daemon mode and is tty) --> + <console>1</console> + </logger> + </clickhouse> + 01-clickhouse-03-query_log.xml: | + <clickhouse> + <query_log replace="1"> + <database>system</database> + <table>query_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </query_log> + </clickhouse> + 01-clickhouse-04-part_log.xml: | + <clickhouse> + <part_log replace="1"> + <database>system</database> + <table>part_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </part_log> + </clickhouse> + 01-clickhouse-05-trace_log.xml: |- + <clickhouse> + <trace_log replace="1"> + <database>system</database> + <table>trace_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </trace_log> + </clickhouse> + 01-clickhouse-06-asynchronous_insert_log.xml: | + <clickhouse> + <asynchronous_insert_log replace="1"> + <database>system</database> + <table>asynchronous_insert_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </asynchronous_insert_log> + </clickhouse> + 01-clickhouse-07-asynchronous_metric_log.xml: | + <clickhouse> + <asynchronous_metric_log replace="1"> + <database>system</database> + <table>asynchronous_metric_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </asynchronous_metric_log> + </clickhouse> + 01-clickhouse-08-backup_log.xml: | + <clickhouse> + <backup_log replace="1"> + <database>system</database> + <table>backup_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </backup_log> + </clickhouse> + 01-clickhouse-09-blob_storage_log.xml: | + <clickhouse> + <blob_storage_log replace="1"> + <database>system</database> + <table>blob_storage_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </blob_storage_log> + </clickhouse> + 01-clickhouse-10-crash_log.xml: | + <clickhouse> + <crash_log replace="1"> + <database>system</database> + <table>crash_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 14 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </crash_log> + </clickhouse> + 01-clickhouse-11-metric_log.xml: | + <clickhouse> + <metric_log replace="1"> + <database>system</database> + <table>metric_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </metric_log> + </clickhouse> + 01-clickhouse-12-query_thread_log.xml: | + <clickhouse> + <query_thread_log replace="1"> + <database>system</database> + <table>query_thread_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </query_thread_log> + </clickhouse> + 01-clickhouse-13-query_views_log.xml: | + <clickhouse> + <query_views_log replace="1"> + <database>system</database> + <table>query_views_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </query_views_log> + </clickhouse> + 01-clickhouse-14-session_log.xml: | + <clickhouse> + <session_log replace="1"> + <database>system</database> + <table>session_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </session_log> + </clickhouse> + 01-clickhouse-15-zookeeper_log.xml: | + <clickhouse> + <zookeeper_log replace="1"> + <database>system</database> + <table>zookeeper_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </zookeeper_log> + </clickhouse> + 01-clickhouse-16-processors_profile_log.xml: | + <clickhouse> + <processors_profile_log replace="1"> + <database>system</database> + <table>processors_profile_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </processors_profile_log> + </clickhouse> +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-etc-configd-files +--- +apiVersion: v1 +data: + config.yaml: |- + annotation: + exclude: + - argocd.argoproj.io/tracking-id + include: [] + clickhouse: + access: + password: "" + port: 8123 + rootCA: "" + scheme: auto + secret: + name: "clickhouse-operator" + namespace: "signoz" + timeouts: + connect: 1 + query: 4 + username: "" + addons: + rules: + - spec: + configuration: + files: null + profiles: null + quotas: null + settings: null + users: null + version: '*' + - spec: + configuration: + files: null + profiles: null + quotas: null + settings: null + users: + '{clickhouseOperatorUser}/access_management': 1 + '{clickhouseOperatorUser}/named_collection_control': 1 + '{clickhouseOperatorUser}/show_named_collections': 1 + '{clickhouseOperatorUser}/show_named_collections_secrets': 1 + version: '>= 23.3' + - spec: + configuration: + files: null + profiles: + clickhouse_operator/format_display_secrets_in_show_and_select: 1 + quotas: null + settings: + display_secrets_in_show_and_select: 1 + users: null + version: '>= 23.5' + configuration: + file: + path: + common: chi/config.d + host: chi/conf.d + user: chi/users.d + network: + hostRegexpTemplate: (chi-{chi}-[^.]+\d+-\d+|clickhouse\-{chi})\.{namespace}\.svc\.cluster\.local$ + user: + default: + networksIP: + - ::1 + - 127.0.0.1 + password: default + profile: default + quota: default + configurationRestartPolicy: + rules: + - rules: + - settings/*: "yes" + - settings/access_control_path: "no" + - settings/dictionaries_config: "no" + - settings/max_server_memory_*: "no" + - settings/max_*_to_drop: "no" + - settings/max_concurrent_queries: "no" + - settings/models_config: "no" + - settings/user_defined_executable_functions_config: "no" + - settings/logger/*: "no" + - settings/macros/*: "no" + - settings/remote_servers/*: "no" + - settings/user_directories/*: "no" + - settings/display_secrets_in_show_and_select: "no" + - zookeeper/*: "yes" + - files/*.xml: "yes" + - files/config.d/*.xml: "yes" + - files/config.d/*dict*.xml: "no" + - files/config.d/*no_restart*: "no" + - profiles/default/background_*_pool_size: "yes" + - profiles/default/max_*_for_server: "yes" + version: '*' + - rules: + - settings/logger: "yes" + version: 21.* + metrics: + timeouts: + collect: 9 + keeper: + configuration: + file: + path: + common: chk/keeper_config.d + host: chk/conf.d + user: chk/users.d + label: + appendScope: "no" + exclude: [] + include: [] + logger: + alsologtostderr: "false" + log_backtrace_at: "" + logtostderr: "true" + stderrthreshold: "" + v: "1" + vmodule: "" + metrics: + labels: + exclude: [] + pod: + terminationGracePeriod: 30 + reconcile: + host: + wait: + exclude: true + include: false + queries: true + replicas: + all: false + delay: 10 + new: true + runtime: + reconcileCHIsThreadsNumber: 10 + reconcileShardsMaxConcurrencyPercent: 50 + reconcileShardsThreadsNumber: 5 + statefulSet: + create: + onFailure: ignore + update: + onFailure: abort + pollInterval: 5 + timeout: 300 + statefulSet: + revisionHistoryLimit: 0 + status: + fields: + action: false + actions: false + error: false + errors: false + template: + chi: + path: chi/templates.d + policy: ApplyOnNextReconcile + chk: + path: chk/templates.d + policy: ApplyOnNextReconcile + watch: + namespaces: + - ".*" +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-etc-files +--- +apiVersion: v1 +data: + 001-templates.json.example: | + { + "apiVersion": "clickhouse.altinity.com/v1", + "kind": "ClickHouseInstallationTemplate", + "metadata": { + "name": "01-default-volumeclaimtemplate" + }, + "spec": { + "templates": { + "volumeClaimTemplates": [ + { + "name": "chi-default-volume-claim-template", + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "2Gi" + } + } + } + } + ], + "podTemplates": [ + { + "name": "chi-default-oneperhost-pod-template", + "distribution": "OnePerHost", + "spec": { + "containers" : [ + { + "name": "clickhouse", + "image": "clickhouse/clickhouse-server:23.8", + "ports": [ + { + "name": "http", + "containerPort": 8123 + }, + { + "name": "client", + "containerPort": 9000 + }, + { + "name": "interserver", + "containerPort": 9009 + } + ] + } + ] + } + } + ] + } + } + } + default-pod-template.yaml.example: | + apiVersion: "clickhouse.altinity.com/v1" + kind: "ClickHouseInstallationTemplate" + metadata: + name: "default-oneperhost-pod-template" + spec: + templates: + podTemplates: + - name: default-oneperhost-pod-template + distribution: "OnePerHost" + default-storage-template.yaml.example: | + apiVersion: "clickhouse.altinity.com/v1" + kind: "ClickHouseInstallationTemplate" + metadata: + name: "default-storage-template-2Gi" + spec: + templates: + volumeClaimTemplates: + - name: default-storage-template-2Gi + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + readme: Templates in this folder are packaged with an operator and available via 'useTemplate' +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-etc-templatesd-files +--- +apiVersion: v1 +data: + 01-clickhouse-user.xml: | + <yandex> + <users> + <clickhouse_operator> + <networks> + <ip>127.0.0.1</ip> + </networks> + <password_sha256_hex></password_sha256_hex> + <profile>clickhouse_operator</profile> + <quota>default</quota> + </clickhouse_operator> + </users> + <profiles> + <clickhouse_operator> + <log_queries>0</log_queries> + <skip_unavailable_shards>1</skip_unavailable_shards> + <http_connection_timeout>10</http_connection_timeout> + </clickhouse_operator> + </profiles> + </yandex> + 02-clickhouse-default-profile.xml: | + <yandex> + <profiles> + <default> + <log_queries>1</log_queries> + <connect_timeout_with_failover_ms>1000</connect_timeout_with_failover_ms> + <distributed_aggregation_memory_efficient>1</distributed_aggregation_memory_efficient> + <parallel_view_processing>1</parallel_view_processing> + </default> + </profiles> + </yandex> + 03-database-ordinary.xml: | + <yandex> + <profiles> + <default> + <default_database_engine>Ordinary</default_database_engine> + </default> + </profiles> + </yandex> +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-etc-usersd-files +--- +apiVersion: v1 +data: null +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-keeper-etc-confd-files +--- +apiVersion: v1 +data: + 01-keeper-01-default-config.xml: | + <!-- IMPORTANT --> + <!-- This file is auto-generated --> + <!-- Do not edit this file - all changes would be lost --> + <!-- Edit appropriate template in the following folder: --> + <!-- deploy/builder/templates-config --> + <!-- IMPORTANT --> + <clickhouse> + <keeper_server> + <coordination_settings> + <min_session_timeout_ms>10000</min_session_timeout_ms> + <operation_timeout_ms>10000</operation_timeout_ms> + <raft_logs_level>information</raft_logs_level> + <session_timeout_ms>100000</session_timeout_ms> + </coordination_settings> + <hostname_checks_enabled>true</hostname_checks_enabled> + <log_storage_path>/var/lib/clickhouse-keeper/coordination/logs</log_storage_path> + <snapshot_storage_path>/var/lib/clickhouse-keeper/coordination/snapshots</snapshot_storage_path> + <storage_path>/var/lib/clickhouse-keeper</storage_path> + <tcp_port>2181</tcp_port> + </keeper_server> + <listen_host>::</listen_host> + <listen_host>0.0.0.0</listen_host> + <listen_try>1</listen_try> + <logger> + <console>1</console> + <level>information</level> + </logger> + <max_connections>4096</max_connections> + </clickhouse> + 01-keeper-02-readiness.xml: | + <!-- IMPORTANT --> + <!-- This file is auto-generated --> + <!-- Do not edit this file - all changes would be lost --> + <!-- Edit appropriate template in the following folder: --> + <!-- deploy/builder/templates-config --> + <!-- IMPORTANT --> + <clickhouse> + <keeper_server> + <http_control> + <port>9182</port> + <readiness> + <endpoint>/ready</endpoint> + </readiness> + </http_control> + </keeper_server> + </clickhouse> + 01-keeper-03-enable-reconfig.xml: |- + <!-- IMPORTANT --> + <!-- This file is auto-generated --> + <!-- Do not edit this file - all changes would be lost --> + <!-- Edit appropriate template in the following folder: --> + <!-- deploy/builder/templates-config --> + <!-- IMPORTANT --> + <clickhouse> + <keeper_server> + <enable_reconfiguration>false</enable_reconfiguration> + </keeper_server> + </clickhouse> +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-keeper-etc-configd-files +--- +apiVersion: v1 +data: + readme: Templates in this folder are packaged with an operator and available via 'useTemplate' +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-keeper-etc-templatesd-files +--- +apiVersion: v1 +data: null +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-keeper-etc-usersd-files diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/deployment.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/deployment.yaml new file mode 100644 index 0000000..a3ce2fe --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/deployment.yaml @@ -0,0 +1,208 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + template: + metadata: + annotations: + checksum/confd-files: 6a503b551565d380d3b4ea4b4c48a66629379558aa6cf0919d55f88f3cb4f56b + checksum/configd-files: a8b2e79da1b016cb2fa1e1b3d61f1edd8872af054a7419040c28b15a236bdc24 + checksum/files: 420251c3f150460f0b1b15211b9c0b965c16342055c2f3a5a57768d229b2ddd0 + checksum/keeper-confd-files: c17fcdc4ce46db171e44a8e5036cafb7173a9806470fafdef717cc3c01318470 + checksum/keeper-configd-files: beed860cfbb0c9e2a0b266f65dee8cbaf22990dc3e200c93b74e697b3cf4c9df + checksum/keeper-templatesd-files: a47abf45edf0f439048648e3d6d1ecede35e560a76f32d8a4aa6055e0fcf58a8 + checksum/keeper-usersd-files: 2fbf34a572990f10fe9bf11c9884c5e847a1a23a02f0728e83bac4f4f9036797 + checksum/templatesd-files: bfbe2e3230fe68164813fcaf74c5cc06b11878c26c764476399ac8803c8bde1c + checksum/usersd-files: c609c91ce94021945657aae781f199517cbc6f2c1f603238c88004f8acf84f95 + clickhouse-operator-metrics/port: "9999" + clickhouse-operator-metrics/scrape: "true" + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + spec: + containers: + - env: + - name: OPERATOR_POD_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OPERATOR_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPERATOR_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: OPERATOR_POD_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: OPERATOR_CONTAINER_CPU_REQUEST + valueFrom: + resourceFieldRef: + containerName: operator + resource: requests.cpu + - name: OPERATOR_CONTAINER_CPU_LIMIT + valueFrom: + resourceFieldRef: + containerName: operator + resource: limits.cpu + - name: OPERATOR_CONTAINER_MEM_REQUEST + valueFrom: + resourceFieldRef: + containerName: operator + resource: requests.memory + - name: OPERATOR_CONTAINER_MEM_LIMIT + valueFrom: + resourceFieldRef: + containerName: operator + resource: limits.memory + image: altinity/clickhouse-operator:0.25.3 + imagePullPolicy: IfNotPresent + name: operator + ports: + - containerPort: 9999 + name: op-metrics + resources: + requests: + cpu: 1000m + memory: 2Gi + securityContext: {} + volumeMounts: + - mountPath: /etc/clickhouse-operator + name: etc-clickhouse-operator-folder + - mountPath: /etc/clickhouse-operator/chi/conf.d + name: etc-clickhouse-operator-confd-folder + - mountPath: /etc/clickhouse-operator/chi/config.d + name: etc-clickhouse-operator-configd-folder + - mountPath: /etc/clickhouse-operator/chi/templates.d + name: etc-clickhouse-operator-templatesd-folder + - mountPath: /etc/clickhouse-operator/chi/users.d + name: etc-clickhouse-operator-usersd-folder + - mountPath: /etc/clickhouse-operator/chk/conf.d + name: etc-keeper-operator-confd-folder + - mountPath: /etc/clickhouse-operator/chk/keeper_config.d + name: etc-keeper-operator-configd-folder + - mountPath: /etc/clickhouse-operator/chk/templates.d + name: etc-keeper-operator-templatesd-folder + - mountPath: /etc/clickhouse-operator/chk/users.d + name: etc-keeper-operator-usersd-folder + - env: + - name: OPERATOR_POD_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OPERATOR_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPERATOR_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: OPERATOR_POD_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: OPERATOR_CONTAINER_CPU_REQUEST + valueFrom: + resourceFieldRef: + containerName: operator + resource: requests.cpu + - name: OPERATOR_CONTAINER_CPU_LIMIT + valueFrom: + resourceFieldRef: + containerName: operator + resource: limits.cpu + - name: OPERATOR_CONTAINER_MEM_REQUEST + valueFrom: + resourceFieldRef: + containerName: operator + resource: requests.memory + - name: OPERATOR_CONTAINER_MEM_LIMIT + valueFrom: + resourceFieldRef: + containerName: operator + resource: limits.memory + image: altinity/metrics-exporter:0.25.3 + imagePullPolicy: IfNotPresent + name: metrics-exporter + ports: + - containerPort: 8888 + name: ch-metrics + resources: + requests: + cpu: 100m + memory: 256Mi + securityContext: {} + volumeMounts: + - mountPath: /etc/clickhouse-operator + name: etc-clickhouse-operator-folder + - mountPath: /etc/clickhouse-operator/chi/conf.d + name: etc-clickhouse-operator-confd-folder + - mountPath: /etc/clickhouse-operator/chi/config.d + name: etc-clickhouse-operator-configd-folder + - mountPath: /etc/clickhouse-operator/chi/templates.d + name: etc-clickhouse-operator-templatesd-folder + - mountPath: /etc/clickhouse-operator/chi/users.d + name: etc-clickhouse-operator-usersd-folder + - mountPath: /etc/clickhouse-operator/chk/conf.d + name: etc-keeper-operator-confd-folder + - mountPath: /etc/clickhouse-operator/chk/keeper_config.d + name: etc-keeper-operator-configd-folder + - mountPath: /etc/clickhouse-operator/chk/templates.d + name: etc-keeper-operator-templatesd-folder + - mountPath: /etc/clickhouse-operator/chk/users.d + name: etc-keeper-operator-usersd-folder + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: signoz-telemetrystore-clickhouse-operator + volumes: + - configMap: + name: signoz-telemetrystore-clickhouse-operator-etc-files + name: etc-clickhouse-operator-folder + - configMap: + name: signoz-telemetrystore-clickhouse-operator-etc-confd-files + name: etc-clickhouse-operator-confd-folder + - configMap: + name: signoz-telemetrystore-clickhouse-operator-etc-configd-files + name: etc-clickhouse-operator-configd-folder + - configMap: + name: signoz-telemetrystore-clickhouse-operator-etc-templatesd-files + name: etc-clickhouse-operator-templatesd-folder + - configMap: + name: signoz-telemetrystore-clickhouse-operator-etc-usersd-files + name: etc-clickhouse-operator-usersd-folder + - configMap: + name: signoz-telemetrystore-clickhouse-operator-keeper-etc-confd-files + name: etc-keeper-operator-confd-folder + - configMap: + name: signoz-telemetrystore-clickhouse-operator-keeper-etc-configd-files + name: etc-keeper-operator-configd-folder + - configMap: + name: signoz-telemetrystore-clickhouse-operator-keeper-etc-templatesd-files + name: etc-keeper-operator-templatesd-folder + - configMap: + name: signoz-telemetrystore-clickhouse-operator-keeper-etc-usersd-files + name: etc-keeper-operator-usersd-folder diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/kustomization.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/kustomization.yaml new file mode 100644 index 0000000..6ed5d3e --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- serviceaccount.yaml +- configmap.yaml +- clusterrole.yaml +- clusterrolebinding.yaml +- service.yaml +- deployment.yaml diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/service.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/service.yaml new file mode 100644 index 0000000..98ebb4f --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator-metrics +spec: + ports: + - name: ch-metrics + port: 8888 + - name: op-metrics + port: 9999 + selector: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/serviceaccount.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/serviceaccount.yaml new file mode 100644 index 0000000..6e3cb19 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/clickhouse-operator/serviceaccount.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/name: signoz-telemetrystore-clickhouse-operator + app.kubernetes.io/version: 0.25.3 + name: signoz-telemetrystore-clickhouse-operator diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/ingester/configmap.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/ingester/configmap.yaml new file mode 100644 index 0000000..c6f483d --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/ingester/configmap.yaml @@ -0,0 +1,144 @@ +apiVersion: v1 +data: + ingester.yaml: |- + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 5s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://signoz-clickhouse:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://signoz-clickhouse:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://signoz-clickhouse:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://signoz-clickhouse:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + metadataexporter: + enabled: true + dsn: tcp://signoz-clickhouse:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + - metadataexporter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + - metadataexporter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + - metadataexporter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: 'server_endpoint: ws://signoz-signoz:4320/v1/opamp' +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-ingester + name: signoz-ingester diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/ingester/deployment.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/ingester/deployment.yaml new file mode 100644 index 0000000..2146976 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/ingester/deployment.yaml @@ -0,0 +1,120 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-ingester + name: signoz-ingester +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-ingester + template: + metadata: + labels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-ingester + spec: + containers: + - args: + - --config=/conf/ingester.yaml + - --manager-config=/conf/opamp.yaml + - --copy-path=/var/tmp/collector-config.yaml + command: + - /signoz-otel-collector + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SIGNOZ_OTEL_COLLECTOR_TIMEOUT + value: 10m + image: signoz/signoz-otel-collector:latest + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 6 + httpGet: + path: / + port: 13133 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: collector + ports: + - containerPort: 4317 + name: otlp-grpc + protocol: TCP + - containerPort: 4318 + name: otlp-http + protocol: TCP + - containerPort: 13133 + name: health + protocol: TCP + readinessProbe: + failureThreshold: 6 + httpGet: + path: / + port: 13133 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + resources: + requests: + cpu: 125m + memory: 512Mi + volumeMounts: + - mountPath: /conf + name: ingester-config-vol + initContainers: + - args: + - migrate + - sync + - check + command: + - /signoz-otel-collector + env: + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_CLUSTER + value: cluster + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_REPLICATION + value: "true" + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN + value: tcp://signoz-clickhouse:9000 + - name: SIGNOZ_OTEL_COLLECTOR_TIMEOUT + value: 10m + image: signoz/signoz-otel-collector:latest + imagePullPolicy: IfNotPresent + name: check + serviceAccountName: signoz-ingester + volumes: + - configMap: + name: signoz-ingester + name: ingester-config-vol diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/ingester/kustomization.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/ingester/kustomization.yaml new file mode 100644 index 0000000..025ce64 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/ingester/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- serviceaccount.yaml +- configmap.yaml +- service.yaml +- deployment.yaml diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/ingester/service.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/ingester/service.yaml new file mode 100644 index 0000000..d26ad2b --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/ingester/service.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-ingester + name: signoz-ingester +spec: + ports: + - name: otlp-grpc + port: 4317 + protocol: TCP + targetPort: 4317 + - name: otlp-http + port: 4318 + protocol: TCP + targetPort: 4318 + - name: health + port: 13133 + protocol: TCP + targetPort: 13133 + selector: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-ingester + type: ClusterIP diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/ingester/serviceaccount.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/ingester/serviceaccount.yaml new file mode 100644 index 0000000..472df92 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/ingester/serviceaccount.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-ingester + name: signoz-ingester diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/kustomization.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/kustomization.yaml new file mode 100644 index 0000000..2f960b3 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: signoz +resources: +- namespace.yaml +- clickhouse-operator +- telemetrystore/clickhouse +- telemetrykeeper/clickhousekeeper +- telemetrystore-migrator +- metastore/postgres +- signoz +- ingester diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/metastore/postgres/kustomization.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/metastore/postgres/kustomization.yaml new file mode 100644 index 0000000..7fcd38a --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/metastore/postgres/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- serviceaccount.yaml +- service.yaml +- statefulset.yaml diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/metastore/postgres/service.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/metastore/postgres/service.yaml new file mode 100644 index 0000000..b858a59 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/metastore/postgres/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: metastore + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-metastore + name: signoz-metastore +spec: + ports: + - name: tcp-postgresql + port: 5432 + protocol: TCP + targetPort: tcp-postgresql + selector: + app.kubernetes.io/component: metastore + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-metastore + type: ClusterIP diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/metastore/postgres/serviceaccount.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/metastore/postgres/serviceaccount.yaml new file mode 100644 index 0000000..f64d89d --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/metastore/postgres/serviceaccount.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: metastore + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-metastore + name: signoz-metastore diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/metastore/postgres/statefulset.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/metastore/postgres/statefulset.yaml new file mode 100644 index 0000000..c5796ae --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/metastore/postgres/statefulset.yaml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/component: metastore + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-metastore + name: signoz-metastore +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: metastore + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-metastore + serviceName: signoz-metastore + template: + metadata: + labels: + app.kubernetes.io/component: metastore + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-metastore + spec: + containers: + - env: + - name: POSTGRESQL_VOLUME_DIR + value: /var/lib/postgresql + - name: POSTGRESQL_PORT_NUMBER + value: "5432" + - name: PGDATA + value: /var/lib/postgresql/data + - name: POSTGRES_DB + value: signoz + - name: POSTGRES_PASSWORD + value: signoz + - name: POSTGRES_USER + value: signoz + image: postgres:16 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - /bin/sh + - -c + - exec pg_isready -U "$POSTGRES_USER" -h 127.0.0.1 -p 5432 + failureThreshold: 6 + initialDelaySeconds: 20 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: postgres + ports: + - containerPort: 5432 + name: tcp-postgresql + protocol: TCP + readinessProbe: + exec: + command: + - /bin/sh + - -c + - exec pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" -h 127.0.0.1 -p + 5432 + failureThreshold: 6 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 3 + resources: + requests: + cpu: 100m + memory: 128Mi + volumeMounts: + - mountPath: /var/lib/postgresql + name: pgdata + serviceAccountName: signoz-metastore + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: pgdata + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/namespace.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/namespace.yaml new file mode 100644 index 0000000..b0391bc --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: signoz diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/signoz/kustomization.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/signoz/kustomization.yaml new file mode 100644 index 0000000..7fcd38a --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/signoz/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- serviceaccount.yaml +- service.yaml +- statefulset.yaml diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/signoz/service.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/signoz/service.yaml new file mode 100644 index 0000000..6ba6318 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/signoz/service.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: signoz + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-signoz + name: signoz-signoz +spec: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: http + - name: opamp + port: 4320 + protocol: TCP + targetPort: opamp + selector: + app.kubernetes.io/component: signoz + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-signoz + type: ClusterIP diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/signoz/serviceaccount.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/signoz/serviceaccount.yaml new file mode 100644 index 0000000..390b499 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/signoz/serviceaccount.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: signoz + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-signoz + name: signoz-signoz diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/signoz/statefulset.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/signoz/statefulset.yaml new file mode 100644 index 0000000..d5c173f --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/signoz/statefulset.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/component: signoz + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-signoz + name: signoz-signoz +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: signoz + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-signoz + serviceName: signoz-signoz + template: + metadata: + labels: + app.kubernetes.io/component: signoz + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-signoz + spec: + containers: + - env: + - name: SIGNOZ_SQLSTORE_POSTGRES_DSN + value: postgres://signoz:signoz@signoz-metastore:5432/signoz?sslmode=disable + - name: SIGNOZ_SQLSTORE_PROVIDER + value: postgres + - name: SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN + value: tcp://signoz-clickhouse:9000 + - name: SIGNOZ_TELEMETRYSTORE_PROVIDER + value: clickhouse + image: signoz/signoz:latest + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 6 + httpGet: + path: /api/v1/health + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: signoz + ports: + - containerPort: 8080 + name: http + protocol: TCP + - containerPort: 4320 + name: opamp + protocol: TCP + readinessProbe: + failureThreshold: 6 + httpGet: + path: /api/v1/health + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + resources: + requests: + cpu: 100m + memory: 200Mi + serviceAccountName: signoz-signoz diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/telemetrykeeper/clickhousekeeper/clickhousekeeperinstallation.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/telemetrykeeper/clickhousekeeper/clickhousekeeperinstallation.yaml new file mode 100644 index 0000000..7b1be82 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/telemetrykeeper/clickhousekeeper/clickhousekeeperinstallation.yaml @@ -0,0 +1,49 @@ +apiVersion: clickhouse-keeper.altinity.com/v1 +kind: ClickHouseKeeperInstallation +metadata: + name: signoz-clickhouse-keeper +spec: + configuration: + clusters: + - layout: + replicasCount: 1 + name: cluster + settings: + keeper_server/coordination_settings/raft_logs_level: information + keeper_server/four_letter_word_white_list: '*' + listen_host: 0.0.0.0 + logger/console: "true" + logger/level: trace + defaults: + templates: + dataVolumeClaimTemplate: default + podTemplate: default + templates: + podTemplates: + - metadata: + labels: + app.kubernetes.io/component: telemetrykeeper + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-clickhouse-keeper + app.kubernetes.io/part-of: signoz + name: default + spec: + containers: + - image: clickhouse/clickhouse-keeper:25.5.6 + imagePullPolicy: IfNotPresent + name: clickhouse-keeper + resources: + requests: + cpu: 1 + memory: 1Gi + securityContext: + fsGroup: 101 + volumeClaimTemplates: + - name: default + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/telemetrykeeper/clickhousekeeper/kustomization.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/telemetrykeeper/clickhousekeeper/kustomization.yaml new file mode 100644 index 0000000..e2d3083 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/telemetrykeeper/clickhousekeeper/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- clickhousekeeperinstallation.yaml diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/telemetrystore-migrator/job.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/telemetrystore-migrator/job.yaml new file mode 100644 index 0000000..d816940 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/telemetrystore-migrator/job.yaml @@ -0,0 +1,99 @@ +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/component: schema-migrator + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-telemetrystore-migrator + name: signoz-telemetrystore-migrator +spec: + backoffLimit: 6 + template: + metadata: + labels: + app.kubernetes.io/component: schema-migrator + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-telemetrystore-migrator + spec: + containers: + - args: + - migrate + - async + - up + env: + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_CLUSTER + value: cluster + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_REPLICATION + value: "true" + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN + value: tcp://signoz-clickhouse:9000 + - name: SIGNOZ_OTEL_COLLECTOR_TIMEOUT + value: 10m + image: signoz/signoz-otel-collector:latest + imagePullPolicy: IfNotPresent + name: async + resources: + requests: + cpu: 10m + memory: 16Mi + initContainers: + - args: + - migrate + - ready + env: + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_CLUSTER + value: cluster + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_REPLICATION + value: "true" + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN + value: tcp://signoz-clickhouse:9000 + - name: SIGNOZ_OTEL_COLLECTOR_TIMEOUT + value: 10m + image: signoz/signoz-otel-collector:latest + imagePullPolicy: IfNotPresent + name: ready + resources: + requests: + cpu: 10m + memory: 16Mi + - args: + - migrate + - bootstrap + env: + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_CLUSTER + value: cluster + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_REPLICATION + value: "true" + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN + value: tcp://signoz-clickhouse:9000 + - name: SIGNOZ_OTEL_COLLECTOR_TIMEOUT + value: 10m + image: signoz/signoz-otel-collector:latest + imagePullPolicy: IfNotPresent + name: bootstrap + resources: + requests: + cpu: 10m + memory: 16Mi + - args: + - migrate + - sync + - up + env: + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_CLUSTER + value: cluster + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_REPLICATION + value: "true" + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN + value: tcp://signoz-clickhouse:9000 + - name: SIGNOZ_OTEL_COLLECTOR_TIMEOUT + value: 10m + image: signoz/signoz-otel-collector:latest + imagePullPolicy: IfNotPresent + name: sync + resources: + requests: + cpu: 10m + memory: 16Mi + restartPolicy: OnFailure diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/telemetrystore-migrator/kustomization.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/telemetrystore-migrator/kustomization.yaml new file mode 100644 index 0000000..ac2f8e0 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/telemetrystore-migrator/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- job.yaml diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/telemetrystore/clickhouse/clickhouseinstallation.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/telemetrystore/clickhouse/clickhouseinstallation.yaml new file mode 100644 index 0000000..2b8e432 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/telemetrystore/clickhouse/clickhouseinstallation.yaml @@ -0,0 +1,300 @@ +apiVersion: clickhouse.altinity.com/v1 +kind: ClickHouseInstallation +metadata: + labels: + app.kubernetes.io/component: telemetrystore + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-clickhouse + app.kubernetes.io/part-of: signoz + app.kubernetes.io/version: 25.5.6 + name: signoz-clickhouse +spec: + configuration: + clusters: + - layout: + replicasCount: 0 + shardsCount: 1 + name: cluster + files: + config.d/crash.yaml: | + send_crash_reports: + enabled: false + config.d/extra.yaml: | + asynchronous_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE + dictionaries_config: '*_dictionary.xml' + format_schema_path: /etc/clickhouse-server/config.d/ + metric_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE + part_log: + ttl: event_date + INTERVAL 1 DAY DELETE + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE + prometheus: + endpoint: /metrics + port: 9363 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: event_date + INTERVAL 1 DAY DELETE + query_thread_log: + ttl: event_date + INTERVAL 1 DAY DELETE + query_views_log: + ttl: event_date + INTERVAL 1 DAY DELETE + session_log: + ttl: event_date + INTERVAL 1 DAY DELETE + tmp_path: /var/lib/clickhouse/tmp/ + trace_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE + user_defined_executable_functions_config: /etc/clickhouse-server/functions/custom-functions.xml + user_directories: + users_xml: + path: users.xml + user_files_path: /var/lib/clickhouse/user_files/ + user_scripts_path: /var/lib/clickhouse/user_scripts/ + zookeeper_log: + ttl: event_date + INTERVAL 1 DAY DELETE + config.d/formatting.yaml: | + logger: + console: 1 + count: 10 + formatting: + names: + date_time: date_time + level: level + logger_name: logger_name + message: message + query_id: query_id + source_file: source_file + source_line: source_line + thread_id: thread_id + thread_name: thread_name + type: json + level: information + size: 1000M + config.d/load.yaml: | + async_load_databases: false + config.d/path.yaml: | + merge_tree: + storage_policy: data + path: /var/lib/clickhouse/ + config.d/storage.yaml: | + storage_configuration: + disks: + data0: + keep_free_space_bytes: 10485760 + path: /mnt/data-0/ + default: {} + policies: + data: + volumes: + data0: + disk: data0 + default: + volumes: + default: + disk: default + config.d/system_log.yaml: | + error_log: + ttl: event_date + INTERVAL 7 DAY DELETE + latency_log: + ttl: event_date + INTERVAL 3 DAY DELETE + query_metric_log: + ttl: event_date + INTERVAL 3 DAY DELETE + text_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 3 DAY DELETE + events.proto: | + syntax = "proto3"; + message Event { + string uuid = 1; + string event = 2; + string properties = 3; + string timestamp = 4; + uint64 team_id = 5; + string distinct_id = 6; + string created_at = 7; + string elements_chain = 8; + } + profiles: + admin/max_threads: 16 + admin/query_plan_max_limit_for_lazy_materialization: "0" + admin/secondary_indices_enable_bulk_filtering: "0" + default/allow_experimental_window_functions: "1" + default/allow_nondeterministic_mutations: "1" + default/allow_simdjson: 0 + default/load_balancing: random + default/log_queries: 1 + default/max_threads: 16 + default/query_plan_max_limit_for_lazy_materialization: "0" + default/secondary_indices_enable_bulk_filtering: "0" + quotas: + default/interval/duration: 3600 + default/interval/errors: 0 + default/interval/execution_time: 0 + default/interval/queries: 0 + default/interval/read_rows: 0 + default/interval/result_rows: 0 + users: + admin/networks/ip: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 172.16.0.0/12 + - 192.0.0.0/24 + - 198.18.0.0/15 + - 192.168.0.0/16 + admin/password: 27ff0399-0d3a-4bd8-919d-17c2181e6fb9 + admin/profile: default + admin/quota: default + default/access_management: 1 + default/named_collection_control: 1 + default/networks/ip: ::/0 + default/password: "" + default/profile: default + default/quota: default + default/show_named_collection: 1 + default/show_named_collection_secrets: 1 + zookeeper: + nodes: + - host: keeper-signoz-clickhouse-keeper + port: 2181 + defaults: + templates: + podTemplate: default + serviceTemplate: default + templates: + podTemplates: + - metadata: + labels: + app.kubernetes.io/component: telemetrystore + app.kubernetes.io/instance: signoz + app.kubernetes.io/name: signoz-clickhouse + app.kubernetes.io/version: 25.5.6 + name: default + spec: + containers: + - command: + - /bin/bash + - -c + - /usr/bin/clickhouse-server --config-file=/etc/clickhouse-server/config.xml + image: clickhouse/clickhouse-server:25.5.6 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 10 + httpGet: + path: /ping + port: http + initialDelaySeconds: 60 + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 1 + name: clickhouse + ports: + - containerPort: 8123 + name: http + - containerPort: 9000 + name: client + - containerPort: 9009 + name: interserver + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ping + port: http + initialDelaySeconds: 10 + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 500m + memory: 512Mi + startupProbe: + failureThreshold: 30 + httpGet: + path: /ping + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /var/lib/clickhouse/user_scripts + name: shared-binary-volume + - mountPath: /etc/clickhouse-server/functions + name: custom-functions-volume + - mountPath: /var/lib/clickhouse + name: default + - mountPath: /mnt/data-0 + name: data-0 + initContainers: + - command: + - sh + - -c + - | + set -e + version="v0.0.1" + node_os=$(uname -s | tr '[:upper:]' '[:lower:]') + node_arch=$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) + echo "Fetching histogram-binary for ${node_os}/${node_arch}" + cd /tmp + wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F${version}/histogram-quantile_${node_os}_${node_arch}.tar.gz" + tar -xzf histogram-quantile.tar.gz + chmod +x histogram-quantile + mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile + echo "histogram-quantile installed successfully" + image: docker.io/alpine:3.18.2 + imagePullPolicy: IfNotPresent + name: histogram + volumeMounts: + - mountPath: /var/lib/clickhouse/user_scripts + name: shared-binary-volume + securityContext: + fsGroup: 101 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 101 + runAsUser: 101 + volumes: + - emptyDir: {} + name: shared-binary-volume + - configMap: + name: signoz-clickhouse-custom-functions + name: custom-functions-volume + serviceTemplates: + - generateName: signoz-clickhouse + metadata: + labels: + app.kubernetes.io/component: telemetrystore + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-clickhouse + app.kubernetes.io/part-of: signoz + name: default + spec: + ports: + - name: http + port: 8123 + - name: tcp + port: 9000 + type: ClusterIP + volumeClaimTemplates: + - name: data-0 + reclaimPolicy: Retain + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 25Gi + - name: default + reclaimPolicy: Retain + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/telemetrystore/clickhouse/configmap.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/telemetrystore/clickhouse/configmap.yaml new file mode 100644 index 0000000..ec8f548 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/telemetrystore/clickhouse/configmap.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +data: + custom-functions.xml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: telemetrystore + app.kubernetes.io/instance: signoz + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: signoz-clickhouse + app.kubernetes.io/part-of: signoz + name: signoz-clickhouse-custom-functions diff --git a/docs/examples/kubernetes/kustomize/pours/deployment/telemetrystore/clickhouse/kustomization.yaml b/docs/examples/kubernetes/kustomize/pours/deployment/telemetrystore/clickhouse/kustomization.yaml new file mode 100644 index 0000000..a6835a5 --- /dev/null +++ b/docs/examples/kubernetes/kustomize/pours/deployment/telemetrystore/clickhouse/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- configmap.yaml +- clickhouseinstallation.yaml diff --git a/docs/examples/railway/README.md b/docs/examples/railway/README.md new file mode 100644 index 0000000..e2eda63 --- /dev/null +++ b/docs/examples/railway/README.md @@ -0,0 +1,5 @@ +# Railway + +| Flavor | Description | +| --- | --- | +| [template](template/) | Deploy SigNoz on Railway using a template | diff --git a/docs/examples/railway/template/README.md b/docs/examples/railway/template/README.md new file mode 100644 index 0000000..e97c518 --- /dev/null +++ b/docs/examples/railway/template/README.md @@ -0,0 +1,69 @@ +# Railway Template + +| Field | Value | +| --- | --- | +| **Mode** | `-` | +| **Flavor** | `template` | +| **Platform** | `railway` | + +## Overview + +Generates per-component Dockerfiles, `railway.json` service definitions, and config files for deploying SigNoz on Railway. Deployment is manual via the Railway dashboard. + +> [!NOTE] +> `foundryctl cast` does not deploy to Railway automatically. It generates the files and prints instructions for manual deployment. + +## Prerequisites + +- A [Railway](https://railway.app) account + +## Configuration + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + flavor: template + platform: railway +``` + +## Deploy + +```bash +# Generate the template files +foundryctl forge -f casting.yaml +``` + +After forging, use the generated files in `pours/deployment/` to create services on Railway. + +## Generated output + +```text +pours/deployment/ + telemetrykeeper/ + Dockerfile + railway.json + keeper.d/ + telemetrystore/ + Dockerfile + railway.json + config.d/ + metastore/ + Dockerfile + railway.json + signoz/ + Dockerfile + railway.json + ingester/ + Dockerfile + railway.json + migrator/ + Dockerfile + railway.json +``` + +## Customization + +For changes to the generated files, use [patches](../../../concepts/patches.md). diff --git a/docs/examples/railway/template/casting.yaml b/docs/examples/railway/template/casting.yaml new file mode 100644 index 0000000..bf6aef1 --- /dev/null +++ b/docs/examples/railway/template/casting.yaml @@ -0,0 +1,8 @@ +apiVersion: v1alpha1 +kind: Installation +metadata: + name: signoz +spec: + deployment: + flavor: template + platform: railway diff --git a/docs/examples/railway/template/casting.yaml.lock b/docs/examples/railway/template/casting.yaml.lock new file mode 100644 index 0000000..620037d --- /dev/null +++ b/docs/examples/railway/template/casting.yaml.lock @@ -0,0 +1,674 @@ +apiVersion: v1alpha1 +kind: Installation +metadata: + name: signoz +spec: + deployment: + flavor: template + platform: railway + ingester: + spec: + cluster: + replicas: 1 + config: + data: + ingester.yaml: | + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 5s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://signoz-telemetrystore-clickhouse.railway.internal:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://signoz-telemetrystore-clickhouse.railway.internal:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://signoz-telemetrystore-clickhouse.railway.internal:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://signoz-telemetrystore-clickhouse.railway.internal:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + metadataexporter: + enabled: true + dsn: tcp://signoz-telemetrystore-clickhouse.railway.internal:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + - metadataexporter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + - metadataexporter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + - metadataexporter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: | + server_endpoint: ws://signoz-signoz.railway.internal:4320/v1/opamp + enabled: true + env: + SIGNOZ_OTEL_COLLECTOR_TIMEOUT: 10m + image: signoz/signoz-otel-collector:latest + version: latest + status: + addresses: + otlp: + - tcp://signoz-ingester.railway.internal:4318 + config: + data: + ingester.yaml: | + connectors: + signozmeter: + metrics_flush_interval: 1h + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + processors: + batch: + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s + batch/meter: + send_batch_size: 20000 + send_batch_max_size: 25000 + timeout: 5s + signozspanmetrics/delta: + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + dimensions_cache_size: 100000 + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + - name: signoz.collector.id + - name: service.version + exporters: + clickhousetraces: + datasource: tcp://signoz-telemetrystore-clickhouse.railway.internal:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemetrics: + dsn: tcp://signoz-telemetrystore-clickhouse.railway.internal:9000/signoz_metrics + timeout: 45s + sending_queue: + enabled: false + clickhouselogsexporter: + dsn: tcp://signoz-telemetrystore-clickhouse.railway.internal:9000/signoz_logs + use_new_schema: true + timeout: 45s + sending_queue: + enabled: false + signozclickhousemeter: + dsn: tcp://signoz-telemetrystore-clickhouse.railway.internal:9000/signoz_meter + timeout: 45s + sending_queue: + enabled: false + metadataexporter: + enabled: true + dsn: tcp://signoz-telemetrystore-clickhouse.railway.internal:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory + extensions: + signoz_health_check: + endpoint: "0.0.0.0:13133" + pprof: + endpoint: "0.0.0.0:1777" + service: + telemetry: + logs: + encoding: json + extensions: + - signoz_health_check + - pprof + pipelines: + traces: + receivers: + - otlp + processors: + - signozspanmetrics/delta + - batch + exporters: + - clickhousetraces + - signozmeter + - metadataexporter + metrics: + receivers: + - otlp + processors: + - batch + exporters: + - signozclickhousemetrics + - signozmeter + - metadataexporter + logs: + receivers: + - otlp + processors: + - batch + exporters: + - clickhouselogsexporter + - signozmeter + - metadataexporter + metrics/meter: + receivers: + - signozmeter + processors: + - batch/meter + exporters: + - signozclickhousemeter + opamp.yaml: | + server_endpoint: ws://signoz-signoz.railway.internal:4320/v1/opamp + env: + SIGNOZ_OTEL_COLLECTOR_TIMEOUT: 10m + metastore: + kind: postgres + spec: + cluster: + replicas: 1 + config: {} + enabled: true + env: + POSTGRES_DB: signoz + POSTGRES_PASSWORD: signoz + POSTGRES_USER: signoz + image: postgres:16 + version: "16" + status: + addresses: + dsn: null + config: {} + env: + POSTGRES_DB: signoz + POSTGRES_PASSWORD: signoz + POSTGRES_USER: signoz + signoz: + spec: + cluster: + replicas: 1 + config: {} + enabled: true + env: + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://signoz-telemetrystore-clickhouse.railway.internal:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + image: signoz/signoz:latest + version: latest + status: + addresses: + apiserver: + - tcp://signoz-signoz.railway.internal:8080 + opamp: + - ws://signoz-signoz.railway.internal:4320 + config: {} + env: + SIGNOZ_SQLSTORE_PROVIDER: postgres + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://signoz-telemetrystore-clickhouse.railway.internal:9000 + SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse + telemetrykeeper: + kind: clickhousekeeper + spec: + cluster: + replicas: 1 + config: + data: + keeper-0.yaml: | + keeper_server: + coordination_settings: + force_sync: false + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + snapshot_distance: 100000 + snapshots_to_keep: 3 + four_letter_word_white_list: '*' + http_control: + endpoint: /ready + port: 9182 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: localhost + id: 0 + port: 9234 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 + listen_host: '::' + logger: + console: true + level: information + enabled: true + image: clickhouse/clickhouse-keeper:25.5.6 + version: 25.5.6 + status: + addresses: + client: + - tcp://signoz-telemetrykeeper-clickhousekeeper.railway.internal:9181 + raft: + - tcp://signoz-telemetrykeeper-clickhousekeeper.railway.internal:9234 + config: + data: + keeper-0.yaml: | + keeper_server: + coordination_settings: + force_sync: false + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + snapshot_distance: 100000 + snapshots_to_keep: 3 + four_letter_word_white_list: '*' + http_control: + endpoint: /ready + port: 9182 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: localhost + id: 0 + port: 9234 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 + listen_host: '::' + logger: + console: true + level: information + extras: + _overrides: | + keeper_server: + http_control: + endpoint: /ready + port: 9182 + raft_configuration: + server: + - hostname: localhost + id: 0 + port: 9234 + listen_host: '::' + service_names: signoz-telemetrykeeper-clickhousekeeper + telemetrystore: + kind: clickhouse + spec: + cluster: + replicas: 0 + shards: 1 + config: + data: + config-0-0.yaml: | + asynchronous_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE + dictionaries_config: '*_dictionary.xml' + display_name: cluster + distributed_ddl: + path: /clickhouse/task_queue/ddl + error_log: + ttl: event_date + INTERVAL 1 DAY DELETE + format_schema_path: /var/lib/clickhouse/format_schemas/ + http_port: 8123 + interserver_http_port: 9009 + latency_log: + ttl: event_date + INTERVAL 1 DAY DELETE + listen_host: '::' + logger: + console: 1 + count: 10 + formatting: + type: console + level: information + size: 1000M + macros: + replica: "00" + shard: "00" + metric_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE + part_log: + ttl: event_date + INTERVAL 1 DAY DELETE + path: /var/lib/clickhouse/ + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE + profiles: + default: + allow_simdjson: 0 + load_balancing: random + log_queries: 1 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: event_date + INTERVAL 1 DAY DELETE + query_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE + query_thread_log: + ttl: event_date + INTERVAL 1 DAY DELETE + query_views_log: + ttl: event_date + INTERVAL 1 DAY DELETE + quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 + remote_servers: + cluster: + shard: + - replica: + host: localhost + port: 9000 + session_log: + ttl: event_date + INTERVAL 1 DAY DELETE + tcp_port: 9000 + text_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE + tmp_path: /var/lib/clickhouse/tmp/ + trace_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE + user_defined_executable_functions_config: '*function.yaml' + user_directories: + users_xml: + path: users.xml + user_files_path: /var/lib/clickhouse/user_files/ + user_scripts_path: /etc/clickhouse-server/user_scripts/ + users: + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 + zookeeper: + node: + - host: signoz-telemetrykeeper-clickhousekeeper.railway.internal + port: 9181 + zookeeper_log: + ttl: event_date + INTERVAL 1 DAY DELETE + functions.yaml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable + enabled: true + image: clickhouse/clickhouse-server:25.5.6 + version: 25.5.6 + status: + addresses: + tcp: + - tcp://signoz-telemetrystore-clickhouse.railway.internal:9000 + config: + data: + config-0-0.yaml: | + asynchronous_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE + dictionaries_config: '*_dictionary.xml' + display_name: cluster + distributed_ddl: + path: /clickhouse/task_queue/ddl + error_log: + ttl: event_date + INTERVAL 1 DAY DELETE + format_schema_path: /var/lib/clickhouse/format_schemas/ + http_port: 8123 + interserver_http_port: 9009 + latency_log: + ttl: event_date + INTERVAL 1 DAY DELETE + listen_host: '::' + logger: + console: 1 + count: 10 + formatting: + type: console + level: information + size: 1000M + macros: + replica: "00" + shard: "00" + metric_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE + part_log: + ttl: event_date + INTERVAL 1 DAY DELETE + path: /var/lib/clickhouse/ + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE + profiles: + default: + allow_simdjson: 0 + load_balancing: random + log_queries: 1 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: event_date + INTERVAL 1 DAY DELETE + query_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE + query_thread_log: + ttl: event_date + INTERVAL 1 DAY DELETE + query_views_log: + ttl: event_date + INTERVAL 1 DAY DELETE + quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 + remote_servers: + cluster: + shard: + - replica: + host: localhost + port: 9000 + session_log: + ttl: event_date + INTERVAL 1 DAY DELETE + tcp_port: 9000 + text_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE + tmp_path: /var/lib/clickhouse/tmp/ + trace_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE + user_defined_executable_functions_config: '*function.yaml' + user_directories: + users_xml: + path: users.xml + user_files_path: /var/lib/clickhouse/user_files/ + user_scripts_path: /etc/clickhouse-server/user_scripts/ + users: + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 + zookeeper: + node: + - host: signoz-telemetrykeeper-clickhousekeeper.railway.internal + port: 9181 + zookeeper_log: + ttl: event_date + INTERVAL 1 DAY DELETE + functions.yaml: | + functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable + extras: + _overrides: | + listen_host: '::' + remote_servers: + cluster: + shard: + - replica: + host: localhost + port: 9000 + user_scripts_path: /etc/clickhouse-server/user_scripts/ + service_names: signoz-telemetrystore-clickhouse diff --git a/docs/examples/railway/template/pours/deployment/ingester/Dockerfile b/docs/examples/railway/template/pours/deployment/ingester/Dockerfile new file mode 100644 index 0000000..16a9446 --- /dev/null +++ b/docs/examples/railway/template/pours/deployment/ingester/Dockerfile @@ -0,0 +1,9 @@ +FROM signoz/signoz-otel-collector:latest + +COPY ingester.yaml /etc/otel-collector-config.yaml +COPY opamp.yaml /etc/opamp-config.yaml + + +ENV SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN=tcp://signoz-telemetrystore-clickhouse.railway.internal:9000 + +ENV SIGNOZ_OTEL_COLLECTOR_TIMEOUT=10m \ No newline at end of file diff --git a/docs/examples/railway/template/pours/deployment/ingester/ingester.yaml b/docs/examples/railway/template/pours/deployment/ingester/ingester.yaml new file mode 100644 index 0000000..83a60ba --- /dev/null +++ b/docs/examples/railway/template/pours/deployment/ingester/ingester.yaml @@ -0,0 +1,132 @@ +connectors: + signozmeter: + dimensions: + - name: service.name + - name: deployment.environment + - name: host.name + metrics_flush_interval: 1h +exporters: + clickhouselogsexporter: + dsn: tcp://signoz-telemetrystore-clickhouse.railway.internal:9000/signoz_logs + sending_queue: + enabled: false + timeout: 45s + use_new_schema: true + clickhousetraces: + datasource: tcp://signoz-telemetrystore-clickhouse.railway.internal:9000/signoz_traces + low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} + sending_queue: + enabled: false + timeout: 45s + use_new_schema: true + metadataexporter: + cache: + provider: in_memory + dsn: tcp://signoz-telemetrystore-clickhouse.railway.internal:9000/signoz_metadata + enabled: true + timeout: 45s + signozclickhousemeter: + dsn: tcp://signoz-telemetrystore-clickhouse.railway.internal:9000/signoz_meter + sending_queue: + enabled: false + timeout: 45s + signozclickhousemetrics: + dsn: tcp://signoz-telemetrystore-clickhouse.railway.internal:9000/signoz_metrics + sending_queue: + enabled: false + timeout: 45s +extensions: + pprof: + endpoint: 0.0.0.0:1777 + signoz_health_check: + endpoint: 0.0.0.0:13133 +processors: + batch: + send_batch_max_size: 55000 + send_batch_size: 50000 + timeout: 5s + batch/meter: + send_batch_max_size: 25000 + send_batch_size: 20000 + timeout: 5s + signozspanmetrics/delta: + aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + dimensions: + - default: default + name: service.namespace + - default: default + name: deployment.environment + - name: signoz.collector.id + - name: service.version + dimensions_cache_size: 100000 + enable_exp_histogram: true + latency_histogram_buckets: + - 100us + - 1ms + - 2ms + - 6ms + - 10ms + - 50ms + - 100ms + - 250ms + - 500ms + - 1000ms + - 1400ms + - 2000ms + - 5s + - 10s + - 20s + - 40s + - 60s + metrics_exporter: signozclickhousemetrics + metrics_flush_interval: 60s +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +service: + extensions: + - signoz_health_check + - pprof + pipelines: + logs: + exporters: + - clickhouselogsexporter + - signozmeter + - metadataexporter + processors: + - batch + receivers: + - otlp + metrics: + exporters: + - signozclickhousemetrics + - signozmeter + - metadataexporter + processors: + - batch + receivers: + - otlp + metrics/meter: + exporters: + - signozclickhousemeter + processors: + - batch/meter + receivers: + - signozmeter + traces: + exporters: + - clickhousetraces + - signozmeter + - metadataexporter + processors: + - signozspanmetrics/delta + - batch + receivers: + - otlp + telemetry: + logs: + encoding: json diff --git a/docs/examples/railway/template/pours/deployment/ingester/opamp.yaml b/docs/examples/railway/template/pours/deployment/ingester/opamp.yaml new file mode 100644 index 0000000..4741f1b --- /dev/null +++ b/docs/examples/railway/template/pours/deployment/ingester/opamp.yaml @@ -0,0 +1 @@ +server_endpoint: ws://signoz-signoz.railway.internal:4320/v1/opamp diff --git a/docs/examples/railway/template/pours/deployment/ingester/railway.json b/docs/examples/railway/template/pours/deployment/ingester/railway.json new file mode 100644 index 0000000..d41645f --- /dev/null +++ b/docs/examples/railway/template/pours/deployment/ingester/railway.json @@ -0,0 +1,12 @@ +{ + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "Dockerfile" + }, + "deploy": { + "numReplicas": 1, + "startCommand": "/bin/sh -c '/signoz-otel-collector migrate sync check && /signoz-otel-collector --config=/etc/otel-collector-config.yaml --manager-config=/etc/opamp-config.yaml --copy-path=/var/tmp/collector-config.yaml'", + "restartPolicyType": "ON_FAILURE", + "restartPolicyMaxRetries": 5 + } +} diff --git a/docs/examples/railway/template/pours/deployment/signoz/Dockerfile b/docs/examples/railway/template/pours/deployment/signoz/Dockerfile new file mode 100644 index 0000000..fe4fe7f --- /dev/null +++ b/docs/examples/railway/template/pours/deployment/signoz/Dockerfile @@ -0,0 +1,4 @@ +FROM signoz/signoz:latest +ENV SIGNOZ_SQLSTORE_PROVIDER=postgres +ENV SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN=tcp://signoz-telemetrystore-clickhouse.railway.internal:9000 +ENV SIGNOZ_TELEMETRYSTORE_PROVIDER=clickhouse \ No newline at end of file diff --git a/docs/examples/railway/template/pours/deployment/signoz/railway.json b/docs/examples/railway/template/pours/deployment/signoz/railway.json new file mode 100644 index 0000000..44648b0 --- /dev/null +++ b/docs/examples/railway/template/pours/deployment/signoz/railway.json @@ -0,0 +1,13 @@ +{ + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "Dockerfile" + }, + "deploy": { + "numReplicas": 1, + "healthcheckPath": "/api/v1/health", + "healthcheckTimeout": 300, + "restartPolicyType": "ON_FAILURE", + "restartPolicyMaxRetries": 5 + } +} diff --git a/docs/examples/railway/template/pours/deployment/telemetrykeeper/Dockerfile b/docs/examples/railway/template/pours/deployment/telemetrykeeper/Dockerfile new file mode 100644 index 0000000..5182bc3 --- /dev/null +++ b/docs/examples/railway/template/pours/deployment/telemetrykeeper/Dockerfile @@ -0,0 +1,26 @@ +FROM clickhouse/clickhouse-keeper:25.5.6 + +# Copy all keeper config files +COPY keeper.d/ /etc/clickhouse-keeper/ +# Create entrypoint script to dynamically select config based on replica index +RUN cat > /entrypoint.sh <<'EOF' +#!/bin/bash +set -euo pipefail + +# Extract replica index from service name, default to 0 +REPLICA_INDEX=$(echo "${RAILWAY_SERVICE_NAME:-}" | grep -oE "[0-9]+$" || echo "0") +CONFIG_FILE="/etc/clickhouse-keeper/keeper-${REPLICA_INDEX}.yaml" + +# Fallback to keeper-0.yaml if replica-specific config doesn't exist +if [ ! -f "$CONFIG_FILE" ]; then + echo "Config $CONFIG_FILE not found, falling back to keeper-0.yaml" + CONFIG_FILE="/etc/clickhouse-keeper/keeper-0.yaml" +fi + +echo "Starting ClickHouse Keeper with config: $CONFIG_FILE" +exec /usr/bin/clickhouse-keeper --config-file="$CONFIG_FILE" "$@" +EOF + +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/docs/examples/railway/template/pours/deployment/telemetrykeeper/keeper.d/keeper-0.yaml b/docs/examples/railway/template/pours/deployment/telemetrykeeper/keeper.d/keeper-0.yaml new file mode 100644 index 0000000..1d49025 --- /dev/null +++ b/docs/examples/railway/template/pours/deployment/telemetrykeeper/keeper.d/keeper-0.yaml @@ -0,0 +1,25 @@ +keeper_server: + coordination_settings: + force_sync: false + operation_timeout_ms: 10000 + raft_logs_level: warning + session_timeout_ms: 30000 + snapshot_distance: 100000 + snapshots_to_keep: 3 + four_letter_word_white_list: '*' + http_control: + endpoint: /ready + port: 9182 + log_storage_path: /var/lib/clickhouse/coordination/log + raft_configuration: + server: + - hostname: localhost + id: 0 + port: 9234 + server_id: 0 + snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots + tcp_port: 9181 +listen_host: '::' +logger: + console: true + level: information diff --git a/docs/examples/railway/template/pours/deployment/telemetrykeeper/railway.json b/docs/examples/railway/template/pours/deployment/telemetrykeeper/railway.json new file mode 100644 index 0000000..84435fb --- /dev/null +++ b/docs/examples/railway/template/pours/deployment/telemetrykeeper/railway.json @@ -0,0 +1,14 @@ +{ + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "Dockerfile" + }, + "deploy": { + "numReplicas": 1, + "healthcheckPath": "/ready", + "healthcheckTimeout": 300, + "restartPolicyType": "ON_FAILURE", + "restartPolicyMaxRetries": 5, + "requiredMountPath": "/var/lib/clickhouse-keeper/" + } +} diff --git a/docs/examples/railway/template/pours/deployment/telemetrystore-migrator/Dockerfile b/docs/examples/railway/template/pours/deployment/telemetrystore-migrator/Dockerfile new file mode 100644 index 0000000..989b75a --- /dev/null +++ b/docs/examples/railway/template/pours/deployment/telemetrystore-migrator/Dockerfile @@ -0,0 +1,6 @@ +FROM signoz/signoz-otel-collector:latest + + +ENV SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN=tcp://signoz-telemetrystore-clickhouse.railway.internal:9000 + +ENV SIGNOZ_OTEL_COLLECTOR_TIMEOUT=10m \ No newline at end of file diff --git a/docs/examples/railway/template/pours/deployment/telemetrystore-migrator/railway.json b/docs/examples/railway/template/pours/deployment/telemetrystore-migrator/railway.json new file mode 100644 index 0000000..840ad30 --- /dev/null +++ b/docs/examples/railway/template/pours/deployment/telemetrystore-migrator/railway.json @@ -0,0 +1,11 @@ +{ + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "Dockerfile" + }, + "deploy": { + "numReplicas": 1, + "startCommand": "/bin/sh -c '/signoz-otel-collector migrate ready && /signoz-otel-collector migrate bootstrap && /signoz-otel-collector migrate sync up && /signoz-otel-collector migrate async up'", + "restartPolicyType": "NEVER" + } +} diff --git a/docs/examples/railway/template/pours/deployment/telemetrystore/Dockerfile b/docs/examples/railway/template/pours/deployment/telemetrystore/Dockerfile new file mode 100644 index 0000000..901fe26 --- /dev/null +++ b/docs/examples/railway/template/pours/deployment/telemetrystore/Dockerfile @@ -0,0 +1,36 @@ +FROM clickhouse/clickhouse-server:25.5.6 + +COPY config.d/ /etc/clickhouse-server/ + +RUN mkdir -p /etc/clickhouse-server/user_scripts/ \ + && chown clickhouse:clickhouse /etc/clickhouse-server/user_scripts + +ARG VERSION=v0.0.1 + +ENV CLICKHOUSE_CONFIG=/etc/clickhouse-server/config-0-0.yaml +ENV CLICKHOUSE_SKIP_USER_SETUP=1 + +# Download and install histogram-quantile binary +RUN cat > /tmp/install-histogram-quantile.sh <<'EOF' +#!/bin/bash +set -euo pipefail + +node_os=$(uname -s | tr '[:upper:]' '[:lower:]') +node_arch=$(uname -m | sed 's/aarch64/arm64/' | sed 's/x86_64/amd64/') + +echo "Fetching histogram-quantile for ${node_os}/${node_arch}" + +cd /tmp +wget -O histogram-quantile.tar.gz \ + "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F${VERSION}/histogram-quantile_${node_os}_${node_arch}.tar.gz" + +tar -xzf histogram-quantile.tar.gz +mv histogram-quantile /etc/clickhouse-server/user_scripts/histogramQuantile +rm histogram-quantile.tar.gz + +echo "histogram-quantile installed successfully" +EOF + +RUN chmod +x /tmp/install-histogram-quantile.sh \ + && /tmp/install-histogram-quantile.sh \ + && rm /tmp/install-histogram-quantile.sh \ No newline at end of file diff --git a/docs/examples/railway/template/pours/deployment/telemetrystore/config.d/config-0-0.yaml b/docs/examples/railway/template/pours/deployment/telemetrystore/config.d/config-0-0.yaml new file mode 100644 index 0000000..909e55a --- /dev/null +++ b/docs/examples/railway/template/pours/deployment/telemetrystore/config.d/config-0-0.yaml @@ -0,0 +1,96 @@ +asynchronous_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE +dictionaries_config: '*_dictionary.xml' +display_name: cluster +distributed_ddl: + path: /clickhouse/task_queue/ddl +error_log: + ttl: event_date + INTERVAL 1 DAY DELETE +format_schema_path: /var/lib/clickhouse/format_schemas/ +http_port: 8123 +interserver_http_port: 9009 +latency_log: + ttl: event_date + INTERVAL 1 DAY DELETE +listen_host: '::' +logger: + console: 1 + count: 10 + formatting: + type: console + level: information + size: 1000M +macros: + replica: "00" + shard: "00" +metric_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE +part_log: + ttl: event_date + INTERVAL 1 DAY DELETE +path: /var/lib/clickhouse/ +processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE +profiles: + default: + allow_simdjson: 0 + load_balancing: random + log_queries: 1 +query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: event_date + INTERVAL 1 DAY DELETE +query_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE +query_thread_log: + ttl: event_date + INTERVAL 1 DAY DELETE +query_views_log: + ttl: event_date + INTERVAL 1 DAY DELETE +quotas: + default: + interval: + duration: 3600 + errors: 0 + execution_time: 0 + queries: 0 + read_rows: 0 + result_rows: 0 +remote_servers: + cluster: + shard: + - replica: + host: localhost + port: 9000 +session_log: + ttl: event_date + INTERVAL 1 DAY DELETE +tcp_port: 9000 +text_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE +tmp_path: /var/lib/clickhouse/tmp/ +trace_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE +user_defined_executable_functions_config: '*function.yaml' +user_directories: + users_xml: + path: users.xml +user_files_path: /var/lib/clickhouse/user_files/ +user_scripts_path: /etc/clickhouse-server/user_scripts/ +users: + default: + access_management: 1 + named_collection_control: 1 + networks: + ip: ::/0 + password: "" + profile: default + quota: default + show_named_collection: 1 + show_named_collection_secrets: 1 +zookeeper: + node: + - host: signoz-telemetrykeeper-clickhousekeeper.railway.internal + port: 9181 +zookeeper_log: + ttl: event_date + INTERVAL 1 DAY DELETE diff --git a/docs/examples/railway/template/pours/deployment/telemetrystore/config.d/functions.yaml b/docs/examples/railway/template/pours/deployment/telemetrystore/config.d/functions.yaml new file mode 100644 index 0000000..9a88469 --- /dev/null +++ b/docs/examples/railway/template/pours/deployment/telemetrystore/config.d/functions.yaml @@ -0,0 +1,13 @@ +functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable diff --git a/docs/examples/railway/template/pours/deployment/telemetrystore/railway.json b/docs/examples/railway/template/pours/deployment/telemetrystore/railway.json new file mode 100644 index 0000000..2137a49 --- /dev/null +++ b/docs/examples/railway/template/pours/deployment/telemetrystore/railway.json @@ -0,0 +1,14 @@ +{ + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "Dockerfile" + }, + "deploy": { + "numReplicas": 1, + "healthcheckPath": "/ping", + "healthcheckTimeout": 300, + "restartPolicyType": "ON_FAILURE", + "restartPolicyMaxRetries": 5, + "requiredMountPath": "/var/lib/clickhouse/" + } +} diff --git a/docs/examples/render/README.md b/docs/examples/render/README.md new file mode 100644 index 0000000..aceafd5 --- /dev/null +++ b/docs/examples/render/README.md @@ -0,0 +1,5 @@ +# Render + +| Flavor | Description | +| --- | --- | +| [blueprint](blueprint/) | Deploy SigNoz on Render using a Blueprint spec | diff --git a/docs/examples/render/blueprint/README.md b/docs/examples/render/blueprint/README.md new file mode 100644 index 0000000..339e4b4 --- /dev/null +++ b/docs/examples/render/blueprint/README.md @@ -0,0 +1,59 @@ +# Render Blueprint + +| Field | Value | +| --- | --- | +| **Mode** | `-` | +| **Flavor** | `blueprint` | +| **Platform** | `render` | + +## Overview + +Generates a Render Blueprint (`render.yaml`) and supporting Dockerfiles for deploying SigNoz on the Render cloud platform. Deployment is manual via Render's Infrastructure as Code flow. + +> [!NOTE] +> `foundryctl cast` does not deploy to Render automatically. It generates the files and prints instructions for manual deployment. + +## Prerequisites + +- A [Render](https://render.com) account + +## Configuration + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + flavor: blueprint + platform: render +``` + +## Deploy + +```bash +# Generate the blueprint and supporting files +foundryctl forge -f casting.yaml +``` + +After forging, deploy the generated `render.yaml` to Render using [Infrastructure as Code](https://render.com/docs/infrastructure-as-code#setup). + +## Generated output + +```text +pours/deployment/ + render.yaml + configs/ + telemetrykeeper/ + Dockerfile + keeper.d/ + telemetrystore/ + Dockerfile + config.d/ + ingester/ + Dockerfile +``` + +## Customization + +For changes to the generated `render.yaml`, use [patches](../../../concepts/patches.md). diff --git a/docs/examples/render/blueprint/casting.yaml b/docs/examples/render/blueprint/casting.yaml index 1390262..5ddfde0 100644 --- a/docs/examples/render/blueprint/casting.yaml +++ b/docs/examples/render/blueprint/casting.yaml @@ -1,4 +1,5 @@ apiVersion: v1alpha1 +kind: Installation metadata: name: o11y spec: diff --git a/docs/examples/render/blueprint/casting.yaml.lock b/docs/examples/render/blueprint/casting.yaml.lock index 3104c34..4e9d22d 100644 --- a/docs/examples/render/blueprint/casting.yaml.lock +++ b/docs/examples/render/blueprint/casting.yaml.lock @@ -1,4 +1,5 @@ apiVersion: v1alpha1 +kind: Installation metadata: name: o11y spec: @@ -28,9 +29,9 @@ spec: endpoint: "0.0.0.0:4318" processors: batch: - send_batch_size: 20000 - send_batch_max_size: 25000 - timeout: 1s + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s batch/meter: send_batch_size: 20000 send_batch_max_size: 25000 @@ -90,6 +91,12 @@ spec: timeout: 45s sending_queue: enabled: false + metadataexporter: + enabled: true + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory extensions: o11y_health_check: endpoint: "0.0.0.0:13133" @@ -163,9 +170,9 @@ spec: endpoint: "0.0.0.0:4318" processors: batch: - send_batch_size: 20000 - send_batch_max_size: 25000 - timeout: 1s + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s batch/meter: send_batch_size: 20000 send_batch_max_size: 25000 @@ -225,6 +232,12 @@ spec: timeout: 45s sending_queue: enabled: false + metadataexporter: + enabled: true + dsn: tcp://signoz-telemetrystore-clickhouse-0-0:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory extensions: o11y_health_check: endpoint: "0.0.0.0:13133" @@ -334,6 +347,9 @@ spec: operation_timeout_ms: 10000 raft_logs_level: warning session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 log_storage_path: /var/lib/clickhouse/coordination/log raft_configuration: server: @@ -365,6 +381,9 @@ spec: operation_timeout_ms: 10000 raft_logs_level: warning session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 log_storage_path: /var/lib/clickhouse/coordination/log raft_configuration: server: @@ -384,7 +403,7 @@ spec: shards: 1 config: data: - config.yaml: | + config-0-0.yaml: | path: /var/lib/clickhouse/ tmp_path: /var/lib/clickhouse/tmp/ user_files_path: /var/lib/clickhouse/user_files/ @@ -404,13 +423,13 @@ spec: level: information size: 1000M macros: - replica: "01" - shard: "01" + replica: "00" + shard: "00" profiles: default: + allow_simdjson: 0 load_balancing: random log_queries: 1 - max_memory_usage: 10000000000 quotas: default: interval: @@ -433,6 +452,40 @@ spec: node: - host: o11y-telemetrykeeper-clickhousekeeper-0 port: 9181 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_thread_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_views_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + part_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + metric_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + asynchronous_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + trace_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + error_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + latency_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + session_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + text_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + zookeeper_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" tcp_port: 9000 user_defined_executable_functions_config: '*function.yaml' user_scripts_path: /var/lib/clickhouse/user_scripts/ @@ -470,7 +523,7 @@ spec: - tcp://o11y-telemetrystore-clickhouse-0-0:9000 config: data: - config.yaml: | + config-0-0.yaml: | path: /var/lib/clickhouse/ tmp_path: /var/lib/clickhouse/tmp/ user_files_path: /var/lib/clickhouse/user_files/ @@ -490,13 +543,13 @@ spec: level: information size: 1000M macros: - replica: "01" - shard: "01" + replica: "00" + shard: "00" profiles: default: + allow_simdjson: 0 load_balancing: random log_queries: 1 - max_memory_usage: 10000000000 quotas: default: interval: @@ -519,6 +572,40 @@ spec: node: - host: o11y-telemetrykeeper-clickhousekeeper-0 port: 9181 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_thread_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_views_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + part_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + metric_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + asynchronous_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + trace_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + error_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + latency_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + session_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + text_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + zookeeper_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" tcp_port: 9000 user_defined_executable_functions_config: '*function.yaml' user_scripts_path: /var/lib/clickhouse/user_scripts/ diff --git a/docs/examples/render/blueprint/pours/deployment/configs/ingester/ingester.yaml b/docs/examples/render/blueprint/pours/deployment/configs/ingester/ingester.yaml index f7df397..4cfa3da 100644 --- a/docs/examples/render/blueprint/pours/deployment/configs/ingester/ingester.yaml +++ b/docs/examples/render/blueprint/pours/deployment/configs/ingester/ingester.yaml @@ -36,9 +36,9 @@ extensions: endpoint: 0.0.0.0:13133 processors: batch: - send_batch_max_size: 25000 - send_batch_size: 20000 - timeout: 1s + send_batch_max_size: 55000 + send_batch_size: 50000 + timeout: 5s batch/meter: send_batch_max_size: 25000 send_batch_size: 20000 diff --git a/docs/examples/render/blueprint/pours/deployment/configs/telemetrykeeper/keeper.d/keeper-0.yaml b/docs/examples/render/blueprint/pours/deployment/configs/telemetrykeeper/keeper.d/keeper-0.yaml index e1ca33d..1606da9 100644 --- a/docs/examples/render/blueprint/pours/deployment/configs/telemetrykeeper/keeper.d/keeper-0.yaml +++ b/docs/examples/render/blueprint/pours/deployment/configs/telemetrykeeper/keeper.d/keeper-0.yaml @@ -1,8 +1,11 @@ keeper_server: coordination_settings: + force_sync: false operation_timeout_ms: 10000 raft_logs_level: warning session_timeout_ms: 30000 + snapshot_distance: 100000 + snapshots_to_keep: 3 four_letter_word_white_list: '*' log_storage_path: /var/lib/clickhouse/coordination/log raft_configuration: diff --git a/docs/examples/render/blueprint/pours/deployment/configs/telemetrystore/Dockerfile b/docs/examples/render/blueprint/pours/deployment/configs/telemetrystore/Dockerfile index f654d9a..045069f 100644 --- a/docs/examples/render/blueprint/pours/deployment/configs/telemetrystore/Dockerfile +++ b/docs/examples/render/blueprint/pours/deployment/configs/telemetrystore/Dockerfile @@ -29,4 +29,28 @@ EOF RUN chmod +x /tmp/install-histogram-quantile.sh \ && /tmp/install-histogram-quantile.sh \ - && rm /tmp/install-histogram-quantile.sh \ No newline at end of file + && rm /tmp/install-histogram-quantile.sh + +# Create wrapper entrypoint to alias the right per-node config as config.yaml +RUN cat > /foundry-entrypoint.sh <<'EOF' +#!/bin/bash +set -euo pipefail + +# Extract shard-replica indices from service name (e.g., signoz-telemetrystore-clickhouse-0-1) +INDICES=$(echo "${RENDER_SERVICE_NAME:-}" | grep -oE "[0-9]+-[0-9]+$" || echo "0-0") +CONFIG_SRC="/etc/clickhouse-server/config-${INDICES}.yaml" + +if [ ! -f "$CONFIG_SRC" ]; then + echo "Config $CONFIG_SRC not found, falling back to config-0-0.yaml" + CONFIG_SRC="/etc/clickhouse-server/config-0-0.yaml" +fi + +ln -sf "$CONFIG_SRC" /etc/clickhouse-server/config.yaml +echo "ClickHouse starting with config: $CONFIG_SRC (aliased as config.yaml)" + +exec /entrypoint.sh "$@" +EOF + +RUN chmod +x /foundry-entrypoint.sh + +ENTRYPOINT ["/foundry-entrypoint.sh"] \ No newline at end of file diff --git a/docs/examples/render/blueprint/pours/deployment/configs/telemetrystore/config.d/config.yaml b/docs/examples/render/blueprint/pours/deployment/configs/telemetrystore/config.d/config-0-0.yaml similarity index 54% rename from docs/examples/render/blueprint/pours/deployment/configs/telemetrystore/config.d/config.yaml rename to docs/examples/render/blueprint/pours/deployment/configs/telemetrystore/config.d/config-0-0.yaml index 38bdaff..8e9ac45 100644 --- a/docs/examples/render/blueprint/pours/deployment/configs/telemetrystore/config.d/config.yaml +++ b/docs/examples/render/blueprint/pours/deployment/configs/telemetrystore/config.d/config-0-0.yaml @@ -1,10 +1,16 @@ +asynchronous_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE dictionaries_config: '*_dictionary.xml' display_name: cluster distributed_ddl: path: /clickhouse/task_queue/ddl +error_log: + ttl: event_date + INTERVAL 1 DAY DELETE format_schema_path: /var/lib/clickhouse/format_schemas/ http_port: 8123 interserver_http_port: 9009 +latency_log: + ttl: event_date + INTERVAL 1 DAY DELETE listen_host: 0.0.0.0 logger: console: 1 @@ -14,14 +20,32 @@ logger: level: information size: 1000M macros: - replica: "01" - shard: "01" + replica: "00" + shard: "00" +metric_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE +part_log: + ttl: event_date + INTERVAL 1 DAY DELETE path: /var/lib/clickhouse/ +processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE profiles: default: + allow_simdjson: 0 load_balancing: random log_queries: 1 - max_memory_usage: 10000000000 +query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: event_date + INTERVAL 1 DAY DELETE +query_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE +query_thread_log: + ttl: event_date + INTERVAL 1 DAY DELETE +query_views_log: + ttl: event_date + INTERVAL 1 DAY DELETE quotas: default: interval: @@ -37,8 +61,16 @@ remote_servers: - replica: host: o11y-telemetrystore-clickhouse-0-0 port: 9000 +session_log: + ttl: event_date + INTERVAL 1 DAY DELETE tcp_port: 9000 +text_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE tmp_path: /var/lib/clickhouse/tmp/ +trace_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE user_defined_executable_functions_config: '*function.yaml' user_directories: users_xml: @@ -60,3 +92,5 @@ zookeeper: node: - host: o11y-telemetrykeeper-clickhousekeeper-0 port: 9181 +zookeeper_log: + ttl: event_date + INTERVAL 1 DAY DELETE diff --git a/docs/examples/render/blueprint/pours/deployment/render.yaml b/docs/examples/render/blueprint/pours/deployment/render.yaml index 9a22454..21542d1 100644 --- a/docs/examples/render/blueprint/pours/deployment/render.yaml +++ b/docs/examples/render/blueprint/pours/deployment/render.yaml @@ -39,6 +39,8 @@ services: - dockerContext: configs/ingester dockerfilePath: configs/ingester/Dockerfile envVars: + - key: SIGNOZ_OTEL_COLLECTOR_TIMEOUT + value: 10m - fromService: name: o11y-telemetrystore-clickhouse-0-0 property: host diff --git a/docs/examples/systemd/README.md b/docs/examples/systemd/README.md new file mode 100644 index 0000000..5dd9472 --- /dev/null +++ b/docs/examples/systemd/README.md @@ -0,0 +1,5 @@ +# Systemd + +| Flavor | Description | +| --- | --- | +| [binary](binary/) | Bare metal Linux with systemd services | diff --git a/docs/examples/systemd/binary/README.md b/docs/examples/systemd/binary/README.md index 1cb3857..e7216f2 100644 --- a/docs/examples/systemd/binary/README.md +++ b/docs/examples/systemd/binary/README.md @@ -60,9 +60,24 @@ sudo chown -R o11y:o11y /opt/o11y /var/lib/o11y /opt/ingester /var/lib/ingester Also, make sure that "o11y" user is allowed to transverse to the pours directory. -## Deployment +## Download SigNoz -Create a `casting.yaml` file: +Download the SigNoz release tarball and extract it into `/opt/signoz`: + +```bash +ARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') +sudo mkdir -p /opt/signoz +curl -fsSL "https://github.com/SigNoz/signoz/releases/latest/download/signoz_linux_${ARCH}.tar.gz" \ + | sudo tar -xz --strip-components=1 -C /opt/signoz +``` + +> [!IMPORTANT] +> Extract the full tarball, do not move the `signoz` binary on its own. SigNoz resolves +> the web frontend and notification templates relative to the binary, so `bin/`, `web/`, +> `templates/`, and `conf/` must stay together under `/opt/signoz`. Moving only the binary +> leaves the UI and alert/email templates unresolved. + +## Configuration ```yaml apiVersion: v1alpha1 @@ -74,7 +89,7 @@ spec: mode: systemd ``` -### 1. Verify Prerequisites +## Deploy ```bash foundryctl gauge -f casting.yaml @@ -86,9 +101,10 @@ foundryctl gauge -f casting.yaml sudo foundryctl cast -f casting.yaml ``` -### 3. Verify Services +> [!NOTE] +> `foundryctl cast` requires `sudo` because it manages systemd services, creates system users, and writes to system directories. -Replace `<name>` with your `metadata.name` from `casting.yaml`: +Step-by-step alternative: ```bash systemctl status <name>-o11y.service @@ -98,18 +114,46 @@ systemctl status <name>-telemetrykeeper-clickhousekeeper-0.service systemctl status <name>-metastore-postgres.service ``` -View logs: +## Generated output + +```text +pours/deployment/ + signoz-ingester.service + signoz-metastore-postgres.service + signoz-signoz.service + signoz-telemetrykeeper-clickhousekeeper-0.service + signoz-telemetrystore-clickhouse-0-0.service + signoz-telemetrystore-migrator.service + configs/ + ingester/ + ingester.yaml + opamp.yaml + telemetrykeeper/ + keeper-0.yaml + telemetrystore/ + config.yaml + functions.yaml +``` + +## After deployment + +Check service status (replace `signoz` with your `metadata.name`): ```bash journalctl -u <name>-o11y.service -f ``` +View logs for a specific service: -## Configuration +```bash +journalctl -u signoz-signoz.service -f +``` -### Custom Binary Path +View logs for all SigNoz services: -Use annotations to specify custom binary paths or other deployment metadata: +```bash +journalctl -u 'signoz-*' -f +``` | Name | Type | Description | |------|------|-------------| diff --git a/docs/examples/systemd/binary/casting.yaml b/docs/examples/systemd/binary/casting.yaml index a3d3564..c3f1d6b 100644 --- a/docs/examples/systemd/binary/casting.yaml +++ b/docs/examples/systemd/binary/casting.yaml @@ -1,4 +1,5 @@ apiVersion: v1alpha1 +kind: Installation metadata: name: o11y spec: diff --git a/docs/examples/systemd/binary/casting.yaml.lock b/docs/examples/systemd/binary/casting.yaml.lock index f4c1e3e..f11834c 100644 --- a/docs/examples/systemd/binary/casting.yaml.lock +++ b/docs/examples/systemd/binary/casting.yaml.lock @@ -1,4 +1,5 @@ apiVersion: v1alpha1 +kind: Installation metadata: annotations: foundry.o11y.io/ingester-binary-path: /opt/ingester/bin/o11y-otel-collector @@ -32,9 +33,9 @@ spec: endpoint: "0.0.0.0:4318" processors: batch: - send_batch_size: 20000 - send_batch_max_size: 25000 - timeout: 1s + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s batch/meter: send_batch_size: 20000 send_batch_max_size: 25000 @@ -94,6 +95,12 @@ spec: timeout: 45s sending_queue: enabled: false + metadataexporter: + enabled: true + dsn: tcp://localhost:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory extensions: o11y_health_check: endpoint: "0.0.0.0:13133" @@ -167,9 +174,9 @@ spec: endpoint: "0.0.0.0:4318" processors: batch: - send_batch_size: 20000 - send_batch_max_size: 25000 - timeout: 1s + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s batch/meter: send_batch_size: 20000 send_batch_max_size: 25000 @@ -229,6 +236,12 @@ spec: timeout: 45s sending_queue: enabled: false + metadataexporter: + enabled: true + dsn: tcp://localhost:9000/signoz_metadata + timeout: 45s + cache: + provider: in_memory extensions: o11y_health_check: endpoint: "0.0.0.0:13133" @@ -276,6 +289,8 @@ spec: - o11yclickhousemeter opamp.yaml: | server_endpoint: ws://localhost:4320/v1/opamp + env: + SIGNOZ_OTEL_COLLECTOR_TIMEOUT: 10m metastore: kind: postgres spec: @@ -341,6 +356,9 @@ spec: operation_timeout_ms: 10000 raft_logs_level: warning session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 log_storage_path: /var/lib/clickhouse/coordination/log raft_configuration: server: @@ -372,6 +390,9 @@ spec: operation_timeout_ms: 10000 raft_logs_level: warning session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 log_storage_path: /var/lib/clickhouse/coordination/log raft_configuration: server: @@ -389,7 +410,7 @@ spec: shards: 1 config: data: - config.yaml: | + config-0-0.yaml: | path: /var/lib/clickhouse/ tmp_path: /var/lib/clickhouse/tmp/ user_files_path: /var/lib/clickhouse/user_files/ @@ -409,13 +430,13 @@ spec: level: information size: 1000M macros: - replica: "01" - shard: "01" + replica: "00" + shard: "00" profiles: default: + allow_simdjson: 0 load_balancing: random log_queries: 1 - max_memory_usage: 10000000000 quotas: default: interval: @@ -431,13 +452,47 @@ spec: remote_servers: cluster: shard: - - replica: - host: localhost - port: 9000 + - replica: + - host: localhost + port: 9000 zookeeper: node: - host: localhost port: 9181 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_thread_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_views_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + part_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + metric_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + asynchronous_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + trace_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + error_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + latency_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + session_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + text_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + zookeeper_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" tcp_port: 9000 user_defined_executable_functions_config: '*function.yaml' user_scripts_path: /var/lib/clickhouse/user_scripts/ @@ -475,7 +530,7 @@ spec: - tcp://localhost:9000 config: data: - config.yaml: | + config-0-0.yaml: | path: /var/lib/clickhouse/ tmp_path: /var/lib/clickhouse/tmp/ user_files_path: /var/lib/clickhouse/user_files/ @@ -495,13 +550,13 @@ spec: level: information size: 1000M macros: - replica: "01" - shard: "01" + replica: "00" + shard: "00" profiles: default: + allow_simdjson: 0 load_balancing: random log_queries: 1 - max_memory_usage: 10000000000 quotas: default: interval: @@ -517,13 +572,47 @@ spec: remote_servers: cluster: shard: - - replica: - host: localhost - port: 9000 + - replica: + - host: localhost + port: 9000 zookeeper: node: - host: localhost port: 9181 + query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_thread_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + query_views_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + part_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + metric_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + asynchronous_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + trace_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + error_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + latency_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + session_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" + text_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" + zookeeper_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" tcp_port: 9000 user_defined_executable_functions_config: '*function.yaml' user_scripts_path: /var/lib/clickhouse/user_scripts/ diff --git a/docs/examples/systemd/binary/pours/deployment/configs/ingester/ingester.yaml b/docs/examples/systemd/binary/pours/deployment/ingester/ingester.yaml similarity index 97% rename from docs/examples/systemd/binary/pours/deployment/configs/ingester/ingester.yaml rename to docs/examples/systemd/binary/pours/deployment/ingester/ingester.yaml index 792781a..ed4ba8e 100644 --- a/docs/examples/systemd/binary/pours/deployment/configs/ingester/ingester.yaml +++ b/docs/examples/systemd/binary/pours/deployment/ingester/ingester.yaml @@ -36,9 +36,9 @@ extensions: endpoint: 0.0.0.0:13133 processors: batch: - send_batch_max_size: 25000 - send_batch_size: 20000 - timeout: 1s + send_batch_max_size: 55000 + send_batch_size: 50000 + timeout: 5s batch/meter: send_batch_max_size: 25000 send_batch_size: 20000 diff --git a/docs/examples/systemd/binary/pours/deployment/configs/ingester/opamp.yaml b/docs/examples/systemd/binary/pours/deployment/ingester/opamp.yaml similarity index 100% rename from docs/examples/systemd/binary/pours/deployment/configs/ingester/opamp.yaml rename to docs/examples/systemd/binary/pours/deployment/ingester/opamp.yaml diff --git a/docs/examples/systemd/binary/pours/deployment/o11y-telemetrystore-clickhouse-0-0.service b/docs/examples/systemd/binary/pours/deployment/o11y-telemetrystore-clickhouse-0-0.service index be728e9..51a2b1d 100644 --- a/docs/examples/systemd/binary/pours/deployment/o11y-telemetrystore-clickhouse-0-0.service +++ b/docs/examples/systemd/binary/pours/deployment/o11y-telemetrystore-clickhouse-0-0.service @@ -3,7 +3,7 @@ WantedBy=multi-user.target [Service] CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE CAP_NET_BIND_SERVICE -ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config.yaml --pid-file=/run/clickhouse/clickhouse.pid +ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config-0-0.yaml --pid-file=/run/clickhouse/clickhouse.pid Group=clickhouse LimitCORE=infinity LimitNOFILE=500000 diff --git a/docs/examples/systemd/binary/pours/deployment/signoz-ingester.service b/docs/examples/systemd/binary/pours/deployment/signoz-ingester.service new file mode 100644 index 0000000..36617d4 --- /dev/null +++ b/docs/examples/systemd/binary/pours/deployment/signoz-ingester.service @@ -0,0 +1,18 @@ +[Install] +WantedBy=multi-user.target + +[Service] +Environment=SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN=tcp://localhost:9000 +Environment=SIGNOZ_OTEL_COLLECTOR_TIMEOUT=10m +ExecStart=/opt/ingester/bin/signoz-otel-collector --config=%Y/ingester/ingester.yaml --manager-config=%Y/ingester/opamp.yaml --copy-path=/var/lib/ingester/ingester.yaml +ExecStartPre=/opt/ingester/bin/signoz-otel-collector migrate sync check +Group=signoz +KillMode=mixed +Restart=on-failure +RestartSec=5 +Type=simple +User=signoz + +[Unit] +Description=SigNoz OTel Collector +Documentation=https://signoz.io/docs diff --git a/docs/examples/systemd/binary/pours/deployment/signoz-signoz.service b/docs/examples/systemd/binary/pours/deployment/signoz-signoz.service new file mode 100644 index 0000000..495aaa4 --- /dev/null +++ b/docs/examples/systemd/binary/pours/deployment/signoz-signoz.service @@ -0,0 +1,23 @@ +[Install] +WantedBy=multi-user.target + +[Service] +Environment=SIGNOZ_ALERTMANAGER_SIGNOZ_TEMPLATES=/opt/signoz/templates/alertmanager/*.gotmpl +Environment=SIGNOZ_EMAILING_TEMPLATES_DIRECTORY=/opt/signoz/templates/email +Environment=SIGNOZ_SQLSTORE_POSTGRES_DSN=postgres://signoz:signoz@localhost:5432/signoz?sslmode=disable +Environment=SIGNOZ_SQLSTORE_PROVIDER=postgres +Environment=SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN=tcp://localhost:9000 +Environment=SIGNOZ_TELEMETRYSTORE_PROVIDER=clickhouse +Environment=SIGNOZ_WEB_DIRECTORY=/opt/signoz/web +ExecStart=/opt/signoz/bin/signoz server +Group=signoz +KillMode=mixed +Restart=on-failure +RestartSec=5 +Type=simple +User=signoz +WorkingDirectory=/opt/signoz + +[Unit] +Description=SigNoz +Documentation=https://signoz.io/docs diff --git a/docs/examples/systemd/binary/pours/deployment/signoz-telemetrystore-migrator.service b/docs/examples/systemd/binary/pours/deployment/signoz-telemetrystore-migrator.service new file mode 100644 index 0000000..052f0df --- /dev/null +++ b/docs/examples/systemd/binary/pours/deployment/signoz-telemetrystore-migrator.service @@ -0,0 +1,18 @@ +[Install] +WantedBy=multi-user.target + +[Service] +Environment=SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN=tcp://localhost:9000 +Environment=SIGNOZ_OTEL_COLLECTOR_TIMEOUT=10m +ExecStart=/bin/bash -c '/opt/ingester/bin/signoz-otel-collector migrate bootstrap && /opt/ingester/bin/signoz-otel-collector migrate sync up && /opt/ingester/bin/signoz-otel-collector migrate async up' +ExecStartPre=/opt/ingester/bin/signoz-otel-collector migrate ready +Group=signoz +RemainAfterExit=yes +Restart=on-failure +RestartSec=5 +Type=oneshot +User=signoz + +[Unit] +Description=SigNoz TelemetryStore Migrator +Documentation=https://signoz.io/docs diff --git a/docs/examples/systemd/binary/pours/deployment/configs/telemetrykeeper/keeper-0.yaml b/docs/examples/systemd/binary/pours/deployment/telemetrykeeper/clickhousekeeper/keeper-0.yaml similarity index 86% rename from docs/examples/systemd/binary/pours/deployment/configs/telemetrykeeper/keeper-0.yaml rename to docs/examples/systemd/binary/pours/deployment/telemetrykeeper/clickhousekeeper/keeper-0.yaml index 9a3fe35..c74400c 100644 --- a/docs/examples/systemd/binary/pours/deployment/configs/telemetrykeeper/keeper-0.yaml +++ b/docs/examples/systemd/binary/pours/deployment/telemetrykeeper/clickhousekeeper/keeper-0.yaml @@ -1,8 +1,11 @@ keeper_server: coordination_settings: + force_sync: false operation_timeout_ms: 10000 raft_logs_level: warning session_timeout_ms: 30000 + snapshot_distance: 100000 + snapshots_to_keep: 3 four_letter_word_white_list: '*' log_storage_path: /var/lib/clickhouse/coordination/log raft_configuration: diff --git a/docs/examples/systemd/binary/pours/deployment/configs/telemetrystore/config.yaml b/docs/examples/systemd/binary/pours/deployment/telemetrystore/clickhouse/config-0-0.yaml similarity index 52% rename from docs/examples/systemd/binary/pours/deployment/configs/telemetrystore/config.yaml rename to docs/examples/systemd/binary/pours/deployment/telemetrystore/clickhouse/config-0-0.yaml index 5a162b0..18c1be4 100644 --- a/docs/examples/systemd/binary/pours/deployment/configs/telemetrystore/config.yaml +++ b/docs/examples/systemd/binary/pours/deployment/telemetrystore/clickhouse/config-0-0.yaml @@ -1,10 +1,16 @@ +asynchronous_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE dictionaries_config: '*_dictionary.xml' display_name: cluster distributed_ddl: path: /clickhouse/task_queue/ddl +error_log: + ttl: event_date + INTERVAL 1 DAY DELETE format_schema_path: /var/lib/clickhouse/format_schemas/ http_port: 8123 interserver_http_port: 9009 +latency_log: + ttl: event_date + INTERVAL 1 DAY DELETE listen_host: 0.0.0.0 logger: console: 1 @@ -14,14 +20,32 @@ logger: level: information size: 1000M macros: - replica: "01" - shard: "01" + replica: "00" + shard: "00" +metric_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE +part_log: + ttl: event_date + INTERVAL 1 DAY DELETE path: /var/lib/clickhouse/ +processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE profiles: default: + allow_simdjson: 0 load_balancing: random log_queries: 1 - max_memory_usage: 10000000000 +query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: event_date + INTERVAL 1 DAY DELETE +query_metric_log: + ttl: event_date + INTERVAL 1 DAY DELETE +query_thread_log: + ttl: event_date + INTERVAL 1 DAY DELETE +query_views_log: + ttl: event_date + INTERVAL 1 DAY DELETE quotas: default: interval: @@ -35,10 +59,18 @@ remote_servers: cluster: shard: - replica: - host: localhost + - host: localhost port: 9000 +session_log: + ttl: event_date + INTERVAL 1 DAY DELETE tcp_port: 9000 +text_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE tmp_path: /var/lib/clickhouse/tmp/ +trace_log: + flush_interval_milliseconds: 30000 + ttl: event_date + INTERVAL 1 DAY DELETE user_defined_executable_functions_config: '*function.yaml' user_directories: users_xml: @@ -60,3 +92,5 @@ zookeeper: node: - host: localhost port: 9181 +zookeeper_log: + ttl: event_date + INTERVAL 1 DAY DELETE diff --git a/docs/examples/systemd/binary/pours/deployment/telemetrystore/clickhouse/functions.yaml b/docs/examples/systemd/binary/pours/deployment/telemetrystore/clickhouse/functions.yaml new file mode 100644 index 0000000..9a88469 --- /dev/null +++ b/docs/examples/systemd/binary/pours/deployment/telemetrystore/clickhouse/functions.yaml @@ -0,0 +1,13 @@ +functions: + argument: + - name: buckets + type: Array(Float64) + - name: counts + type: Array(Float64) + - name: quantile + type: Array(Float64) + command: ./histogramQuantile + format: CSV + name: histogramQuantile + return_type: Float64 + type: executable diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..5cd465a --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,122 @@ +# Getting Started + +Install `foundryctl` and deploy SigNoz in three steps. + +## 1. Install foundryctl + +The fastest path is the install script, which detects your OS/arch, verifies the +download checksum, installs the binary into `$XDG_BIN_HOME` or `~/.local/bin`, +and prints a PATH hint if needed: + +```bash +curl -fsSL https://signoz.io/foundry.sh | bash +``` + +Pin a specific version: + +```bash +curl -fsSL https://signoz.io/foundry.sh | FOUNDRY_VERSION=v0.1.4 bash +``` + +### Manual install + +If you prefer to download the release archive yourself: + +**Linux:** + +```bash +curl -L "https://github.com/SigNoz/foundry/releases/latest/download/foundry_linux_$(uname -m | sed 's/x86_64/amd64/g' | sed 's/aarch64/arm64/g').tar.gz" -o foundry.tar.gz +tar -xzf foundry.tar.gz +``` + +**macOS:** + +```bash +curl -L "https://github.com/SigNoz/foundry/releases/latest/download/foundry_darwin_$(uname -m | sed 's/x86_64/amd64/g' | sed 's/arm64/arm64/g').tar.gz" -o foundry.tar.gz +tar -xzf foundry.tar.gz +``` + +**Windows (PowerShell):** + +```powershell +$ARCH = if ($env:PROCESSOR_ARCHITECTURE -eq "ARM64") { "arm64" } else { "amd64" } +Invoke-WebRequest -Uri "https://github.com/SigNoz/foundry/releases/latest/download/foundry_windows_${ARCH}.tar.gz" -OutFile foundry.tar.gz -UseBasicParsing +tar -xzf foundry.tar.gz +``` + +The archive extracts to a directory named `foundry_<os>_<arch>` (for example, `foundry_darwin_arm64`) containing `bin/foundryctl`. + +### Add to PATH + +To run `foundryctl` from anywhere, move the binary onto your `PATH`: + +```bash +mkdir -p "$HOME/.local/bin" +mv foundry_*/bin/foundryctl "$HOME/.local/bin/" +``` + +If `~/.local/bin` isn't already on your `PATH` (common on macOS), add it to your shell config: + +```bash +# zsh (~/.zshrc) +export PATH="$HOME/.local/bin:$PATH" + +# bash (~/.bashrc on Linux, ~/.bash_profile on macOS) +export PATH="$HOME/.local/bin:$PATH" + +# fish (~/.config/fish/config.fish) +fish_add_path $HOME/.local/bin +``` + +Reload your shell or `source` the file you edited. + +### Verify + +```bash +foundryctl --help +``` + +## 2. Create a casting + +A casting is a YAML file that describes your SigNoz deployment. Create a file called `casting.yaml`: + +```yaml +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + mode: docker + flavor: compose +``` + +This minimal casting deploys SigNoz using Docker Compose with all default settings. + +> [!TIP] +> Run `foundryctl gen examples` to generate working casting files for every supported deployment mode (Docker, Kubernetes, systemd, Render, and more). + +## 3. Deploy + +```bash +./bin/foundryctl cast -f casting.yaml +``` + +Foundry validates your tools (`gauge`), generates deployment files (`forge`), and deploys SigNoz (`cast`) in one step. + +## Validate + +Check that SigNoz is running: + +```bash +docker ps +``` + +All containers should show `Up` status. Open `http://localhost:8080` to access the SigNoz UI. + +## What's next + +- [Casting concepts](concepts/casting.md) - understand casting files in depth +- [Moldings](concepts/moldings.md) - configure individual components +- [Patches](concepts/patches.md) - customize generated output +- [CLI reference](reference/cli.md) - all commands and flags +- [Examples](examples/) - working examples for Docker, Kubernetes, systemd, and cloud platforms diff --git a/docs/ledger.md b/docs/ledger.md new file mode 100644 index 0000000..8bfe960 --- /dev/null +++ b/docs/ledger.md @@ -0,0 +1,48 @@ +# Ledger + +Foundryctl maintains an anonymous usage ledger to help the SigNoz team understand how the tool is used, identify common errors, and prioritize improvements. **No personally identifiable information (PII) is collected.** + +## What is collected + +Each command execution sends a single event with the following properties: + +| Property | Description | Example | +|---|---|---| +| `platform` | Deployment platform from casting.yaml | `aws`, `docker`, `linux` | +| `mode` | Deployment mode | `docker`, `systemd`, `kubernetes` | +| `flavor` | Deployment flavor | `compose`, `binary`, `helm` | +| `patches_configured` | Whether patches are defined | `true` / `false` | +| `patch_count` | Number of patch entries | `0`, `2` | +| `infrastructure_enabled` | Whether IaC generation is enabled | `true` / `false` | +| `metastore_kind` | MetaStore backend type | `postgres`, `sqlite` | +| `telemetry_store_kind` | TelemetryStore backend type | `clickhouse` | +| `telemetry_keeper_kind` | TelemetryKeeper backend type | `clickhousekeeper` | +| `success` | Whether the command succeeded | `true` / `false` | +| `error` | Error message (on failure only) | `missing tool: docker` | +| `os` | Operating system | `linux`, `darwin` | +| `arch` | CPU architecture | `amd64`, `arm64` | +| `foundry_version` | foundryctl version | `0.1.0` | + +### Identity + +Events are attributed using a hashed machine ID (HMAC-SHA256 of the OS machine ID with an application-specific salt). The hash is not reversible and cannot be correlated across different applications. No usernames, emails, IP addresses, hostnames, or file contents are sent. + +## Tracked commands + +All commands send the same `foundryctl` event, differentiated by the `command` property: + +- `gauge` +- `forge` +- `cast` +- `catalog` + +## How to disable the ledger + +### Per-command + +Use the `--no-ledger` flag on any command: + +```bash +foundryctl forge --no-ledger +foundryctl --no-ledger cast +``` diff --git a/docs/reference/casting-file.md b/docs/reference/casting-file.md new file mode 100644 index 0000000..63fe42f --- /dev/null +++ b/docs/reference/casting-file.md @@ -0,0 +1,168 @@ +# Casting File Reference + +Complete reference for the `casting.yaml` configuration file. For conceptual overview, see [Casting](../concepts/casting.md). + +## Top-level structure + +```yaml +apiVersion: v1alpha1 +metadata: + name: <string> # required + annotations: <map> # optional +spec: + deployment: <deployment> # required + signoz: <molding> + ingester: <molding> + telemetrystore: <molding> + telemetrykeeper: <molding> + metastore: <metastore> + patches: <list> +``` + +## Deployment + +Defines where and how SigNoz is deployed. + +```yaml +spec: + deployment: + mode: <string> + flavor: <string> + platform: <string> +``` + +### Supported combinations + +Each row is a valid combination. Mixing values across rows is not supported. + +| Target | `mode` | `flavor` | `platform` | +| --- | --- | --- | --- | +| Docker Compose | `docker` | `compose` | - | +| Docker Swarm | `docker` | `swarm` | - | +| Systemd (binary) | `systemd` | `binary` | - | +| Kubernetes (Kustomize) | `kubernetes` | `kustomize` | - | +| Kubernetes (Helm) | `kubernetes` | `helm` | - | +| Render | - | `blueprint` | `render` | +| Coolify | - | `stack` | `coolify` | +| Railway | - | `template` | `railway` | +| AWS ECS (EC2) | `ec2` | `terraform` | `ecs` | + +## Molding spec + +Each molding (`signoz`, `ingester`, `telemetrystore`, `telemetrykeeper`) accepts a `spec` block: + +```yaml +<molding>: + spec: + enabled: <bool> + image: <string> + version: <string> + cluster: + replicas: <int> + shards: <int> + env: <map> + config: + data: <map> +``` + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `enabled` | bool | `true` | Include this component in the deployment | +| `image` | string | (built-in) | Container image (Docker and Kubernetes modes) | +| `version` | string | (built-in) | Version label (systemd mode, tagging) | +| `cluster.replicas` | int | `1` | Number of replicas | +| `cluster.shards` | int | `1` | Number of shards (TelemetryStore only) | +| `env` | map | `{}` | Environment variables as key-value pairs | +| `config.data` | map | `{}` | Config file overrides: filename to file contents | + +## MetaStore + +The metastore has an additional `kind` field to select the backend. + +```yaml +spec: + metastore: + kind: <string> + spec: <molding-spec> +``` + +| Kind | Backend | Notes | +| --- | --- | --- | +| `postgres` | PostgreSQL | Default. Recommended for production. | +| `sqlite` | SQLite | Embedded, single-node only. | + +## Patches + +List of patch operations applied to generated output files. + +```yaml +spec: + patches: + - type: <string> + target: <string> + operations: + - op: <string> + path: <string> + value: <any> + from: <string> +``` + +| Field | Required | Description | +| --- | --- | --- | +| `type` | No | Patch driver. Default: `jsonpatch`. | +| `target` | Yes | Output file to patch. Exact path, basename, or glob. | +| `operations` | Yes | List of JSON Patch (RFC 6902) operations. | + +See [Patches](../concepts/patches.md) for operation details and examples. + +## Annotations + +### Systemd binary paths + +Required when using `mode: systemd`, `flavor: binary`. + +| Annotation | Description | +| --- | --- | +| `foundry.signoz.io/signoz-binary-path` | Path to the SigNoz binary | +| `foundry.signoz.io/ingester-binary-path` | Path to the OTel Collector binary | +| `foundry.signoz.io/metastore-postgres-binary-path` | Path to the PostgreSQL binary | + +```yaml +metadata: + name: signoz + annotations: + foundry.signoz.io/signoz-binary-path: /opt/signoz/bin/signoz + foundry.signoz.io/ingester-binary-path: /opt/ingester/bin/signoz-otel-collector + foundry.signoz.io/metastore-postgres-binary-path: /usr/bin/postgres +``` + +### Kubernetes Helm annotations + +Optional. Override the default Helm chart source when using `mode: kubernetes`, `flavor: helm`. + +| Annotation | Default | Description | +| --- | --- | --- | +| `foundry.signoz.io/kubernetes-helm-casting-chart` | `signoz/signoz` | Helm chart reference | +| `foundry.signoz.io/kubernetes-helm-casting-repo-url` | `https://charts.signoz.io` | Helm chart repository URL | +| `foundry.signoz.io/kubernetes-helm-casting-repo-name` | `signoz` | Helm chart repository name | +| `foundry.signoz.io/kubernetes-helm-casting-forge-chart` | - | Set to `true` to download and bundle the chart locally during forge | + +### ECS annotations + +Required when using `platform: ecs`, `mode: ec2`, `flavor: terraform`. + +| Annotation | Maps to tfvar | Description | +| --- | --- | --- | +| `foundry.signoz.io/ecs/region` | `region` | AWS region | +| `foundry.signoz.io/ecs/cluster-id` | `ecs_cluster_id` | ECS cluster ARN or ID | +| `foundry.signoz.io/ecs/subnet-ids` | `subnet_ids` | Comma-separated subnet IDs | +| `foundry.signoz.io/ecs/security-group-ids` | `security_group_ids` | Comma-separated security group IDs | +| `foundry.signoz.io/ecs/vpc-id` | `vpc_id` | VPC ID for Cloud Map namespace | +| `foundry.signoz.io/ecs/config-bucket` | `config_bucket` | S3 bucket for component configs | +| `foundry.signoz.io/ecs/task-role-arn` | `task_role_arn` | IAM role ARN for ECS tasks | +| `foundry.signoz.io/ecs/task-execution-role-arn` | `task_execution_role_arn` | IAM role ARN for task execution | +| `foundry.signoz.io/ecs/capacity-provider` | `capacity_provider` | ECS capacity provider name | + +## Schema + +The full JSON Schema for `casting.yaml` is available at [`docs/schemas/v1alpha1.yaml`](../schemas/v1alpha1.yaml). diff --git a/docs/reference/cli.md b/docs/reference/cli.md new file mode 100644 index 0000000..e6d21b6 --- /dev/null +++ b/docs/reference/cli.md @@ -0,0 +1,80 @@ +# CLI Reference + +`foundryctl` is the Foundry command-line tool for generating and deploying SigNoz. + +## Usage + +``` +foundryctl [command] [flags] +``` + +## Global flags + +| Flag | Description | Default | +|---|---|---| +| `-d`, `--debug` | Enable debug mode with verbose logging | `false` | +| `-f`, `--file` | Path to the casting configuration file | `casting.yaml` | +| `-p`, `--pours` | Directory for generated output | `./pours` | +| `-h`, `--help` | Help for foundryctl | | + +> [!TIP] +> Use `--debug` when troubleshooting errors. It shows internal details that are hidden by default. + +## Commands + +### gauge + +Validate that all required tools are installed for your deployment mode. + +```bash +foundryctl gauge -f casting.yaml +``` + +Exits with an error if any required tool is missing. Run this before `forge` or `cast` to catch missing dependencies early. + +### forge + +Generate deployment and configuration files from your casting. + +```bash +foundryctl forge -f casting.yaml -p ./pours +``` + +Reads the casting file, merges your overrides with defaults, and writes the generated files to the pours directory. The output depends on the deployment mode: Compose files for Docker, service units for systemd, Kubernetes manifests for Kustomize, Helm values for Helm, and so on. + +After forging, a `casting.yaml.lock` file is written with checksums to track the current deployment state. + +### cast + +Deploy SigNoz to your target environment. Runs `gauge` and `forge` automatically before deploying. + +```bash +foundryctl cast -f casting.yaml +``` + +Skip individual steps: + +```bash +# Skip tool validation +foundryctl cast --no-gauge + +# Skip file generation (use existing pours) +foundryctl cast --no-forge +``` + +> [!NOTE] +> Some deployment modes (Render, Coolify, Railway) do not support automated deployment. For these, `cast` generates the files and prints instructions for manual deployment on the target platform. + +### gen + +Generate example casting files and pours for all supported deployment modes. + +```bash +# Generate example castings and forged output +foundryctl gen examples + +# Generate JSON schemas +foundryctl gen schemas +``` + +This is the fastest way to get a working `casting.yaml` for your environment. Each example is written to `docs/examples/<deployment>/` with a minimal casting file and pre-forged output. diff --git a/docs/schemas/v1alpha1.yaml b/docs/schemas/v1alpha1.yaml deleted file mode 100644 index fd66c38..0000000 --- a/docs/schemas/v1alpha1.yaml +++ /dev/null @@ -1,359 +0,0 @@ -definitions: - V1Alpha1CastingSpec: - properties: - deployment: - $ref: '#/definitions/V1Alpha1TypeDeployment' - description: Deployment configuration for the platform - ingester: - $ref: '#/definitions/V1Alpha1Ingester' - description: The configuration for the ingester molding - metastore: - $ref: '#/definitions/V1Alpha1MetaStore' - description: The configuration for the meta store molding - o11y: - $ref: '#/definitions/V1Alpha1Hanzo O11y' - description: The configuration for the Hanzo O11y molding - telemetrykeeper: - $ref: '#/definitions/V1Alpha1TelemetryKeeper' - description: The configuration for the telemetry keeper molding - telemetrystore: - $ref: '#/definitions/V1Alpha1TelemetryStore' - description: The configuration for the telemetry store molding - type: object - V1Alpha1CastingStatus: - properties: - checksum: - description: Checksum of the casting file - type: string - type: object - V1Alpha1Ingester: - properties: - spec: - $ref: '#/definitions/V1Alpha1MoldingSpec' - status: - $ref: '#/definitions/V1Alpha1IngesterStatus' - type: object - V1Alpha1IngesterStatus: - properties: - addresses: - $ref: '#/definitions/V1Alpha1IngesterStatusAddresses' - config: - $ref: '#/definitions/V1Alpha1TypeConfig' - description: Configuration for the molding - env: - additionalProperties: - type: string - description: Environment variables for the molding - type: object - extras: - additionalProperties: - type: string - description: Extra information about the molding - type: object - type: object - V1Alpha1IngesterStatusAddresses: - properties: - otlp: - items: - type: string - type: - - array - - "null" - type: object - V1Alpha1MetaStore: - properties: - kind: - description: Kind of the meta store to use - examples: - - postgres - - sqlite - type: string - spec: - $ref: '#/definitions/V1Alpha1MoldingSpec' - description: Specification for the meta store - status: - $ref: '#/definitions/V1Alpha1MetaStoreStatus' - description: Status of the meta store - type: object - V1Alpha1MetaStoreStatus: - properties: - addresses: - $ref: '#/definitions/V1Alpha1MetaStoreStatusAddresses' - description: Addresses of the meta store - config: - $ref: '#/definitions/V1Alpha1TypeConfig' - description: Configuration for the molding - env: - additionalProperties: - type: string - description: Environment variables for the molding - type: object - extras: - additionalProperties: - type: string - description: Extra information about the molding - type: object - type: object - V1Alpha1MetaStoreStatusAddresses: - properties: - dsn: - description: DSN addresses - items: - type: string - type: - - array - - "null" - type: object - V1Alpha1MoldingSpec: - properties: - cluster: - $ref: '#/definitions/V1Alpha1TypeCluster' - description: Cluster configuration for the molding - config: - $ref: '#/definitions/V1Alpha1TypeConfig' - description: Configuration for the molding - enabled: - default: true - description: Whether the molding is enabled - type: boolean - env: - additionalProperties: - type: string - description: Environment variables for the molding - type: object - image: - description: Container image of the molding - examples: - - ghcr.io/hanzoai/o11y:latest - type: string - version: - description: The version of the molding to use - examples: - - latest - type: string - type: object - V1Alpha1Hanzo O11y: - properties: - spec: - $ref: '#/definitions/V1Alpha1MoldingSpec' - status: - $ref: '#/definitions/V1Alpha1Hanzo O11yStatus' - type: object - V1Alpha1Hanzo O11yStatus: - properties: - addresses: - $ref: '#/definitions/V1Alpha1Hanzo O11yStatusAddresses' - config: - $ref: '#/definitions/V1Alpha1TypeConfig' - description: Configuration for the molding - env: - additionalProperties: - type: string - description: Environment variables for the molding - type: object - extras: - additionalProperties: - type: string - description: Extra information about the molding - type: object - type: object - V1Alpha1Hanzo O11yStatusAddresses: - properties: - apiserver: - items: - type: string - type: - - array - - "null" - opamp: - items: - type: string - type: - - array - - "null" - type: object - V1Alpha1TelemetryKeeper: - properties: - kind: - description: Kind of the telemetry keeper to use - examples: - - clickhousekeeper - type: string - spec: - $ref: '#/definitions/V1Alpha1MoldingSpec' - description: Specification for the telemetry keeper - status: - $ref: '#/definitions/V1Alpha1TelemetryKeeperStatus' - description: Status of the telemetry keeper - type: object - V1Alpha1TelemetryKeeperStatus: - properties: - addresses: - $ref: '#/definitions/V1Alpha1TelemetryKeeperStatusAddresses' - description: Addresses of the telemetry keeper - config: - $ref: '#/definitions/V1Alpha1TypeConfig' - description: Configuration for the molding - env: - additionalProperties: - type: string - description: Environment variables for the molding - type: object - extras: - additionalProperties: - type: string - description: Extra information about the molding - type: object - type: object - V1Alpha1TelemetryKeeperStatusAddresses: - properties: - client: - description: Client addresses - items: - type: string - type: - - array - - "null" - raft: - description: Raft addresses - items: - type: string - type: - - array - - "null" - type: object - V1Alpha1TelemetryStore: - properties: - kind: - description: Kind of the telemetry store to use - examples: - - clickhouse - type: string - spec: - $ref: '#/definitions/V1Alpha1MoldingSpec' - description: Specification for the telemetry store - status: - $ref: '#/definitions/V1Alpha1TelemetryStoreStatus' - description: Status of the telemetry store - type: object - V1Alpha1TelemetryStoreStatus: - properties: - addresses: - $ref: '#/definitions/V1Alpha1TelemetryStoreStatusAddresses' - description: Addresses of the telemetry store - config: - $ref: '#/definitions/V1Alpha1TypeConfig' - description: Configuration for the molding - env: - additionalProperties: - type: string - description: Environment variables for the molding - type: object - extras: - additionalProperties: - type: string - description: Extra information about the molding - type: object - type: object - V1Alpha1TelemetryStoreStatusAddresses: - properties: - tcp: - description: TCP addresses - items: - type: string - type: - - array - - "null" - type: object - V1Alpha1TypeCluster: - properties: - replicas: - description: Number of replicas for the component - examples: - - 1 - type: - - "null" - - integer - shards: - description: Number of shards for the component - examples: - - 1 - type: - - "null" - - integer - type: object - V1Alpha1TypeConfig: - properties: - data: - additionalProperties: - type: string - description: Configuration data as key-value pairs - type: object - type: object - V1Alpha1TypeDeployment: - properties: - flavor: - description: Flavor of mode for the deployment - examples: - - compose - - swarm - - helmfile - - helm - - kustomize - - binary - - rpm - - deb - - chocolatey - type: string - mode: - description: Type of installation method - examples: - - binary - - docker - - kubernetes - - helm - - nomad - - windows - - systemctl - type: string - platform: - description: Provider where an installation runs on - examples: - - aws - - gcp - - azure - - digitalocean - - railway - - docker - - linux - type: string - type: object - V1Alpha1TypeMetadata: - properties: - annotations: - additionalProperties: - type: string - description: Unstructured key-value map for arbitrary metadata - type: object - name: - description: The name of this installation - examples: - - o11y-dev - type: string - type: object -properties: - apiVersion: - description: API Version of the casting configuration schema - examples: - - v1alpha1 - type: string - metadata: - $ref: '#/definitions/V1Alpha1TypeMetadata' - description: Metadata of the casting configuration - spec: - $ref: '#/definitions/V1Alpha1CastingSpec' - description: Specification for the casting - status: - $ref: '#/definitions/V1Alpha1CastingStatus' - description: Status of the casting -type: object diff --git a/docs/standalone/Dockerfile.multi-arch b/docs/standalone/Dockerfile.multi-arch new file mode 100644 index 0000000..275ec81 --- /dev/null +++ b/docs/standalone/Dockerfile.multi-arch @@ -0,0 +1,74 @@ +ARG DEBIAN_SHA="pass-a-valid-docker-sha-otherwise-this-will-fail" + +FROM debian@sha256:${DEBIAN_SHA} +LABEL maintainer="signoz" + +ARG OS="linux" +ARG ARCH + +ARG SIGNOZ_VERSION=latest +ARG INGESTER_VERSION=latest +ARG FOUNDRY_VERSION=latest + +# Install systemd +RUN apt-get update && apt-get install -y --no-install-recommends \ + systemd \ + systemd-sysv \ + ca-certificates \ + curl \ + tar \ + gnupg && \ + rm -f /lib/systemd/system/multi-user.target.wants/* \ + /etc/systemd/system/*.wants/* \ + /lib/systemd/system/local-fs.target.wants/* \ + /lib/systemd/system/sockets.target.wants/*udev* \ + /lib/systemd/system/sockets.target.wants/*initctl* \ + /lib/systemd/system/sysinit.target.wants/systemd-tmpfiles-setup* \ + /lib/systemd/system/systemd-update-utmp* + +# SigNoz +RUN mkdir -p /opt/signoz && \ + curl -fsSL "https://github.com/SigNoz/signoz/releases/${SIGNOZ_VERSION}/download/signoz_${OS}_${ARCH}.tar.gz" | \ + tar -xz --strip-components=1 -C /opt/signoz && \ + chmod +x /opt/signoz/bin/signoz + +# OTel Collector (ingester) +RUN mkdir -p /opt/ingester && \ + curl -fsSL "https://github.com/SigNoz/signoz-otel-collector/releases/${INGESTER_VERSION}/download/signoz-otel-collector_${OS}_${ARCH}.tar.gz" | \ + tar -xz --strip-components=1 -C /opt/ingester && \ + chmod +x /opt/ingester/bin/signoz-otel-collector + +# Foundry +RUN mkdir -p /usr/local/bin && \ + curl -fsSL "https://github.com/SigNoz/foundry/releases/${FOUNDRY_VERSION}/download/foundry_${OS}_${ARCH}.tar.gz" | \ + tar -xz --strip-components=1 -C /tmp && \ + mv /tmp/bin/foundryctl /usr/local/bin/foundryctl && \ + chmod +x /usr/local/bin/foundryctl + + +# Install ClickHouse from apt repo +RUN curl -fsSL 'https://packages.clickhouse.com/rpm/lts/repodata/repomd.xml.key' | \ + gpg --dearmor -o /usr/share/keyrings/clickhouse-keyring.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main" \ + > /etc/apt/sources.list.d/clickhouse.list && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + clickhouse-server \ + clickhouse-client && \ + systemctl disable clickhouse-server || true + +RUN apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN useradd -r -m -d /opt/signoz -s /bin/false signoz && \ + mkdir -p /var/lib/signoz /var/lib/ingester /etc/foundry /opt/signoz/web && \ + chown signoz:signoz /var/lib/signoz /var/lib/ingester /opt/signoz + +COPY docs/standalone/casting.yaml /etc/foundry/casting.yaml +COPY docs/standalone/foundry-setup.service /etc/systemd/system/foundry-setup.service +RUN mkdir -p /var/lib/foundry && systemctl enable foundry-setup.service + +EXPOSE 8080 4317 4318 +VOLUME ["/var/lib/clickhouse", "/var/lib/signoz"] + +STOPSIGNAL SIGRTMIN+3 +CMD ["/sbin/init"] diff --git a/docs/standalone/README.md b/docs/standalone/README.md new file mode 100644 index 0000000..46f9ac3 --- /dev/null +++ b/docs/standalone/README.md @@ -0,0 +1,86 @@ +# Standalone Docker Image + +| | | +| --- | --- | +| **Casting** | `systemd` / `binary` | +| **Use Case** | Single Docker image for quick testing, development, and CI | + +## Overview + +An OpenTelemetry-native observability backend in a single Docker image. The standalone image bundles all SigNoz components into one container for development, demo, and testing environments. + +**Included components:** + +- **SigNoz** - query engine and UI +- **OpenTelemetry Collector** - telemetry ingestion +- **ClickHouse** - telemetry storage +- **SQLite** - metadata storage +- **[Foundry](https://github.com/SigNoz/foundry)** - deployment orchestration via `foundryctl` + +Applications can send telemetry using OpenTelemetry's standard defaults (OTLP gRPC/HTTP) without additional configuration. On first boot, Foundry generates all service configs and starts components via systemd. + +## Prerequisites + +- Docker Engine 20.10+ + +## Deploy + +```bash +docker run -d --name signoz --privileged \ + -p 8080:8080 \ + -p 4317:4317 \ + -p 4318:4318 \ + signoz/signoz-standalone:latest +``` + +Access SigNoz UI at `http://localhost:8080`. + +Send telemetry to: + +- OTLP gRPC: `localhost:4317` +- OTLP HTTP: `localhost:4318` + +## Customization + +To customize the deployment, mount your own `casting.yaml` into the container: + +```bash +docker run -d --name signoz --privileged \ + -p 8080:8080 \ + -p 4317:4317 \ + -p 4318:4318 \ + -v ./casting.yaml:/etc/foundry/casting.yaml \ + signoz/signoz-standalone:latest +``` + +See the default [casting.yaml](casting.yaml) for the full config structure. + +## Persist Data + +```bash +docker run -d --name signoz --privileged \ + -p 8080:8080 \ + -p 4317:4317 \ + -p 4318:4318 \ + -v signoz-clickhouse:/var/lib/clickhouse \ + -v signoz-data:/var/lib/signoz \ + signoz/signoz-standalone:latest +``` + +## After deployment + +```bash +# View logs for all services +docker exec signoz journalctl -f + +# View logs for a specific service +docker exec signoz journalctl -u signoz-signoz.service -f +docker exec signoz journalctl -u signoz-ingester.service -f +docker exec signoz journalctl -u signoz-telemetrystore-clickhouse-0-0.service -f +``` + +## Limitations + +- Requires `--privileged` flag (systemd needs cgroup access) +- `docker logs` is empty - use `journalctl` inside the container +- Single-node only (no clustering) diff --git a/docs/standalone/casting.yaml b/docs/standalone/casting.yaml new file mode 100644 index 0000000..69ba1f9 --- /dev/null +++ b/docs/standalone/casting.yaml @@ -0,0 +1,12 @@ +apiVersion: v1alpha1 +metadata: + name: signoz + annotations: + foundry.signoz.io/signoz-binary-path: /opt/signoz/bin/signoz + foundry.signoz.io/ingester-binary-path: /opt/ingester/bin/signoz-otel-collector +spec: + deployment: + flavor: binary + mode: systemd + metastore: + kind: sqlite diff --git a/docs/standalone/foundry-setup.service b/docs/standalone/foundry-setup.service new file mode 100644 index 0000000..1cab795 --- /dev/null +++ b/docs/standalone/foundry-setup.service @@ -0,0 +1,13 @@ +[Unit] +Description=Foundry Setup - SigNoz Standalone +After=multi-user.target +ConditionPathExists=!/var/lib/foundry/.initialized + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/bin/foundryctl cast -f /etc/foundry/casting.yaml -p /etc/foundry/pours --no-gauge +ExecStartPost=/bin/touch /var/lib/foundry/.initialized + +[Install] +WantedBy=multi-user.target diff --git a/go.mod b/go.mod index 4d2d1d4..f029069 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,11 @@ go 1.26.3 require ( github.com/Masterminds/sprig/v3 v3.3.0 + github.com/denisbrodbeck/machineid v1.0.1 + github.com/evanphx/json-patch/v5 v5.9.11 + github.com/google/jsonschema-go v0.4.3 + github.com/olekukonko/tablewriter v1.1.4 + github.com/segmentio/analytics-go/v3 v3.3.0 github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 github.com/swaggest/jsonschema-go v0.3.79 @@ -37,14 +42,33 @@ require ( github.com/go-openapi/swag/yamlutils v0.25.5 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gosuri/uitable v0.0.4 // indirect + github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/huandu/xstrings v1.5.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect + github.com/olekukonko/errors v1.2.0 // indirect + github.com/olekukonko/ll v0.1.6 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rubenv/sql-migrate v1.8.1 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect + github.com/segmentio/backo-go v1.0.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect @@ -63,6 +87,7 @@ require ( k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31 // indirect k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/kustomize/api v0.20.1 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect ) diff --git a/go.sum b/go.sum index 58e7a46..62db203 100644 --- a/go.sum +++ b/go.sum @@ -6,14 +6,77 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= +github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= +github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bool64/dev v0.2.43 h1:yQ7qiZVef6WtCl2vDYU0Y+qSq+0aBrQzY8KXkklk9cQ= github.com/bool64/dev v0.2.43/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= github.com/bool64/shared v0.1.5 h1:fp3eUhBsrSjNCQPcSdQqZxxh9bBwrYiZ+zOKFkM0/2E= github.com/bool64/shared v0.1.5/go.mod h1:081yz68YC9jeFB3+Bbmno2RFWvGKv1lPKkMP6MHJlPs= +github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70= +github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= +github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= +github.com/clipperhouse/displaywidth v0.10.0 h1:GhBG8WuerxjFQQYeuZAeVTuyxuX+UraiZGD4HJQ3Y8g= +github.com/clipperhouse/displaywidth v0.10.0/go.mod h1:XqJajYsaiEwkxOj4bowCTMcT1SgvHo9flfF3jQasdbs= +github.com/clipperhouse/uax29/v2 v2.6.0 h1:z0cDbUV+aPASdFb2/ndFnS9ts/WNXgTNNGFoKXuhpos= +github.com/clipperhouse/uax29/v2 v2.6.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/containerd/containerd v1.7.30 h1:/2vezDpLDVGGmkUXmlNPLCCNKHJ5BbC5tJB5JNzQhqE= +github.com/containerd/containerd v1.7.30/go.mod h1:fek494vwJClULlTpExsmOyKCMUAbuVjlFsJQc4/j44M= +github.com/containerd/errdefs v0.3.0 h1:FSZgGOeK4yuT/+DnF07/Olde/q4KBoMsaamhXxIMDp4= +github.com/containerd/errdefs v0.3.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= +github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/distribution/distribution/v3 v3.0.0 h1:q4R8wemdRQDClzoNNStftB2ZAfqOiN6UX90KJc4HjyM= +github.com/distribution/distribution/v3 v3.0.0/go.mod h1:tRNuFoZsUdyRVegq8xGNeds4KLjwLCRin/tTo6i1DhU= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo= +github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v5.9.11+incompatible h1:ixHHqfcGvxhWkniF1tWxBHA0yb4Z+d1UQi45df52xW8= +github.com/evanphx/json-patch v5.9.11+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/foxcpp/go-mockdns v1.2.0 h1:omK3OrHRD1IWJz1FuFBCFquhXslXoF17OvBS6JPzZF0= +github.com/foxcpp/go-mockdns v1.2.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ= @@ -61,6 +124,25 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE= +github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= +github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/golang-lru/arc/v2 v2.0.5 h1:l2zaLDubNhW4XO3LnliVj0GXO3+/CGNJAg1dcN2Fpfw= +github.com/hashicorp/golang-lru/arc/v2 v2.0.5/go.mod h1:ny6zBSQZi2JxIeYcv7kt2sH2PXJtirBN7RDhRpxPkxU= +github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= +github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJn+Ichc= @@ -75,8 +157,12 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -85,11 +171,38 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= +github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 h1:EaDatTxkdHG+U3Bk4EUr+DZ7fOGwTfezUiUJMaIcaho= +github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5/go.mod h1:fyalQWdtzDBECAQFBJuQe5bzQ02jGd5Qcbgb97Flm7U= +github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 h1:EfpWLLCyXw8PSM2/XNJLjI3Pb27yVE+gIAfeqp8LUCc= +github.com/redis/go-redis/extra/redisotel/v9 v9.0.5/go.mod h1:WZjPDy7VNzn77AAfnAfVjZNvfJTYfPetfZk5yoSTLaQ= +github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= +github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rubenv/sql-migrate v1.8.1 h1:EPNwCvjAowHI3TnZ+4fQu3a915OpnQoPAjTXCGOy2U0= +github.com/rubenv/sql-migrate v1.8.1/go.mod h1:BTIKBORjzyxZDS6dzoiw6eAFYJ1iNlGAtjn4LGeVjS8= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/segmentio/analytics-go/v3 v3.3.0 h1:8VOMaVGBW03pdBrj1CMFfY9o/rnjJC+1wyQHlVxjw5o= +github.com/segmentio/analytics-go/v3 v3.3.0/go.mod h1:p8owAF8X+5o27jmvUognuXxdtqvSGtD0ZrfY2kcS9bE= +github.com/segmentio/backo-go v1.0.0 h1:kbOAtGJY2DqOR0jfRkYEorx/b18RgtepGtY3+Cpe6qA= +github.com/segmentio/backo-go v1.0.0/go.mod h1:kJ9mm9YmoWSkk+oQ+5Cj8DEoRCX2JT6As4kEtIIOp1M= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= @@ -102,8 +215,12 @@ github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3A github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -125,6 +242,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= @@ -147,6 +266,8 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k= gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -160,6 +281,10 @@ k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 h1:kBawHLSnx/mYHmRnNUf9d4CpjREbe k8s.io/utils v0.0.0-20260319190234-28399d86e0b5/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/kustomize/api v0.20.1 h1:iWP1Ydh3/lmldBnH/S5RXgT98vWYMaTUL1ADcr+Sv7I= +sigs.k8s.io/kustomize/api v0.20.1/go.mod h1:t6hUFxO+Ph0VxIk1sKp1WS0dOjbPCtLJ4p8aADLwqjM= +sigs.k8s.io/kustomize/kyaml v0.21.1 h1:IVlbmhC076nf6foyL6Taw4BkrLuEsXUXNpsE+ScX7fI= +sigs.k8s.io/kustomize/kyaml v0.21.1/go.mod h1:hmxADesM3yUN2vbA5z1/YTBnzLJ1dajdqpQonwBL1FQ= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= diff --git a/internal/casting/casting.go b/internal/casting/casting.go index 1ae8bbc..6a1d2ad 100644 --- a/internal/casting/casting.go +++ b/internal/casting/casting.go @@ -14,11 +14,11 @@ const DeploymentDir = "deployment" type Casting interface { // Returns the enricher for the casting. - Enricher(ctx context.Context, config *v1alpha1.Casting) (molding.MoldingEnricher, error) + Enricher(ctx context.Context, config *installation.Casting) (molding.MoldingEnricher, error) // Generates all the files needed for casting. - Forge(ctx context.Context, config v1alpha1.Casting, poursPath string) ([]types.Material, error) + Forge(ctx context.Context, config installation.Casting, poursPath string) ([]domain.Material, error) // Runs the forged files. - Cast(ctx context.Context, config v1alpha1.Casting, poursPath string) error + Cast(ctx context.Context, config installation.Casting, poursPath string) error } diff --git a/internal/casting/collectionagent/casting.go b/internal/casting/collectionagent/casting.go new file mode 100644 index 0000000..038afff --- /dev/null +++ b/internal/casting/collectionagent/casting.go @@ -0,0 +1,15 @@ +package collectionagent + +import ( + "context" + + "github.com/signoz/foundry/api/v1alpha1/collectionagent" + "github.com/signoz/foundry/internal/domain" + collectionagentmolding "github.com/signoz/foundry/internal/molding/collectionagent" +) + +type Casting interface { + Enricher(ctx context.Context, config *collectionagent.Casting) (collectionagentmolding.MoldingEnricher, error) + Forge(ctx context.Context, config collectionagent.Casting, poursPath string) ([]domain.Material, error) + Cast(ctx context.Context, config collectionagent.Casting, poursPath string) error +} diff --git a/internal/casting/collectionagent/planner.go b/internal/casting/collectionagent/planner.go new file mode 100644 index 0000000..7f6227c --- /dev/null +++ b/internal/casting/collectionagent/planner.go @@ -0,0 +1,99 @@ +package collectionagent + +import ( + "context" + "log/slog" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/collectionagent" + "github.com/signoz/foundry/internal/domain" + foundryerrors "github.com/signoz/foundry/internal/errors" + collectionagentmolding "github.com/signoz/foundry/internal/molding/collectionagent" + "github.com/signoz/foundry/internal/molding/collectionagent/collectormolding" + "github.com/signoz/foundry/internal/planner" + "github.com/signoz/foundry/internal/tooler" +) + +var _ planner.Planner = (*Planner)(nil) + +// Planner is the CollectionAgent Kind's per-Kind orchestrator. It satisfies +// the foundry planner contract by exposing this Kind's moldings, enricher, +// and casting strategy as verbs on a single value. +type Planner struct { + config *collectionagent.Casting + logger *slog.Logger + casting Casting + toolers []tooler.Tooler + enricher collectionagentmolding.MoldingEnricher + moldings []collectionagentmolding.Molding +} + +func NewPlanner(ctx context.Context, c *collectionagent.Casting, logger *slog.Logger) (planner.Planner, error) { + registry := NewRegistry(logger) + + castingStrategy, err := registry.Casting(c.Spec.Deployment) + if err != nil { + return nil, err + } + + toolers, err := registry.Toolers(c.Spec.Deployment) + if err != nil { + return nil, err + } + + enricher, err := castingStrategy.Enricher(ctx, c) + if err != nil { + return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to get molding enricher") + } + + moldings := []collectionagentmolding.Molding{ + collectormolding.New(logger), + } + + return &Planner{ + config: c, + logger: logger, + casting: castingStrategy, + toolers: toolers, + enricher: enricher, + moldings: moldings, + }, nil +} + +func (p *Planner) Machinery() v1alpha1.Machinery { return p.config } +func (p *Planner) Patches() []v1alpha1.PatchEntry { return p.config.Spec.Patches } + +func (p *Planner) MoldingKinds() []v1alpha1.MoldingKind { + kinds := make([]v1alpha1.MoldingKind, len(p.moldings)) + for i, m := range p.moldings { + kinds[i] = m.Kind() + } + return kinds +} + +func (p *Planner) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind) error { + return p.enricher.EnrichStatus(ctx, kind, p.config) +} + +func (p *Planner) Mold(ctx context.Context, kind v1alpha1.MoldingKind) error { + for _, m := range p.moldings { + if m.Kind() == kind { + return m.MoldV1Alpha1(ctx, p.config) + } + } + return foundryerrors.Newf(foundryerrors.TypeInternal, "molding %q not registered for collectionagent planner", kind) +} + +func (p *Planner) MergeStatusIntoSpec() error { + return p.config.MergeStatusIntoSpec() +} + +func (p *Planner) Forge(ctx context.Context, target string) ([]domain.Material, error) { + return p.casting.Forge(ctx, *p.config, target) +} + +func (p *Planner) Cast(ctx context.Context, poursPath string) error { + return p.casting.Cast(ctx, *p.config, poursPath) +} + +func (p *Planner) Toolers() []tooler.Tooler { return p.toolers } diff --git a/internal/casting/collectionagent/registry.go b/internal/casting/collectionagent/registry.go new file mode 100644 index 0000000..0756fdb --- /dev/null +++ b/internal/casting/collectionagent/registry.go @@ -0,0 +1,51 @@ +package collectionagent + +import ( + "log/slog" + + "github.com/signoz/foundry/api/v1alpha1" + foundryerrors "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/tooler" +) + +type CastingItem struct { + Casting Casting + Toolers []tooler.Tooler +} + +type Registry struct { + castings map[v1alpha1.TypeDeployment]CastingItem +} + +func NewRegistry(logger *slog.Logger) *Registry { + return &Registry{ + castings: map[v1alpha1.TypeDeployment]CastingItem{}, + } +} + +func (registry *Registry) lookup(deployment v1alpha1.TypeDeployment) (CastingItem, bool) { + if item, ok := registry.castings[deployment]; ok { + return item, true + } + if deployment.Platform != (v1alpha1.Platform{}) { + item, ok := registry.castings[v1alpha1.TypeDeployment{Mode: deployment.Mode, Flavor: deployment.Flavor}] + return item, ok + } + return CastingItem{}, false +} + +func (registry *Registry) Casting(deployment v1alpha1.TypeDeployment) (Casting, error) { + item, ok := registry.lookup(deployment) + if !ok { + return nil, foundryerrors.Newf(foundryerrors.TypeUnsupported, "collectionagent deployment '%+v' is not supported", deployment) + } + return item.Casting, nil +} + +func (registry *Registry) Toolers(deployment v1alpha1.TypeDeployment) ([]tooler.Tooler, error) { + item, ok := registry.lookup(deployment) + if !ok { + return nil, foundryerrors.Newf(foundryerrors.TypeUnsupported, "collectionagent deployment '%+v' is not supported", deployment) + } + return item.Toolers, nil +} diff --git a/internal/casting/coolifycasting/casting.go b/internal/casting/coolifycasting/casting.go new file mode 100644 index 0000000..e7c5818 --- /dev/null +++ b/internal/casting/coolifycasting/casting.go @@ -0,0 +1,66 @@ +package coolifycasting + +import ( + "bytes" + "context" + "log/slog" + "path/filepath" + + "github.com/signoz/foundry/api/v1alpha1/installation" + rootcasting "github.com/signoz/foundry/internal/casting" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/molding" +) + +var _ rootcasting.Casting = (*coolifyCasting)(nil) + +type coolifyCasting struct { + logger *slog.Logger + castings []*domain.Template +} + +func New(logger *slog.Logger) *coolifyCasting { + return &coolifyCasting{ + logger: logger, + castings: []*domain.Template{ + coolifyYAMLTemplate, + }, + } +} + +func (c *coolifyCasting) Enricher(ctx context.Context, config *installation.Casting) (molding.MoldingEnricher, error) { + return newCoolifyMoldingEnricher(config) +} + +func (c *coolifyCasting) Forge(ctx context.Context, config installation.Casting, poursPath string) ([]domain.Material, error) { + buf := bytes.NewBuffer(nil) + err := coolifyYAMLTemplate.Execute(buf, config) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to execute coolify yaml template") + } + + coolifyMaterial, err := domain.NewYAMLMaterial(buf.Bytes(), filepath.Join(rootcasting.DeploymentDir, "coolify.yaml")) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to create coolify yaml material") + } + + return []domain.Material{coolifyMaterial}, nil +} + +func (c *coolifyCasting) Cast(ctx context.Context, config installation.Casting, poursPath string) error { + c.logger.InfoContext(ctx, "Please run 'forge' first to generate the Coolify Casting", + slog.String("pours_path", poursPath)) + c.logger.InfoContext(ctx, "After forging, deploy coolify.yaml to Coolify using the stack feature", + slog.String("docs", "https://coolify.io/docs/knowledge-base/docker/compose")) + return nil +} + +func getCoolifyMaterial(config *installation.Casting, path string) (domain.StructuredMaterial, error) { + buf := bytes.NewBuffer(nil) + err := coolifyYAMLTemplate.Execute(buf, config) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to execute coolify yaml template") + } + return domain.NewYAMLMaterial(buf.Bytes(), path) +} diff --git a/internal/casting/coolifycasting/embed.go b/internal/casting/coolifycasting/embed.go new file mode 100644 index 0000000..c4c8d24 --- /dev/null +++ b/internal/casting/coolifycasting/embed.go @@ -0,0 +1,14 @@ +package coolifycasting + +import ( + "embed" + + "github.com/signoz/foundry/internal/domain" +) + +//go:embed templates/*.gotmpl +var templates embed.FS + +var ( + coolifyYAMLTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/coolify.yaml.gotmpl", domain.FormatYAML) +) diff --git a/internal/casting/coolifycasting/enricher.go b/internal/casting/coolifycasting/enricher.go new file mode 100644 index 0000000..81101f8 --- /dev/null +++ b/internal/casting/coolifycasting/enricher.go @@ -0,0 +1,115 @@ +package coolifycasting + +import ( + "context" + "path/filepath" + "strings" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/installation" + rootcasting "github.com/signoz/foundry/internal/casting" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/molding" +) + +var _ molding.MoldingEnricher = (*coolifyMoldingEnricher)(nil) + +type coolifyMoldingEnricher struct { + material domain.StructuredMaterial +} + +func newCoolifyMoldingEnricher(config *installation.Casting) (*coolifyMoldingEnricher, error) { + material, err := getCoolifyMaterial(config, filepath.Join(rootcasting.DeploymentDir, "coolify.yaml")) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to get coolify yaml material") + } + return &coolifyMoldingEnricher{material: material}, nil +} + +func (enricher *coolifyMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *installation.Casting) error { + switch kind { + case v1alpha1.MoldingKindTelemetryStore: + containerNames, err := enricher.material.GetStringSlice("services|@keys") + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to get telemetrystore container names") + } + + var telemetrystoreContainerNames []string + for _, containerName := range containerNames { + if strings.Contains(containerName, "telemetrystore-clickhouse") && !strings.Contains(containerName, "user-scripts") { + telemetrystoreContainerNames = append(telemetrystoreContainerNames, domain.MustNewAddress("tcp", containerName, 9000).String()) + } + } + config.Spec.TelemetryStore.Status.Addresses.TCP = telemetrystoreContainerNames + + case v1alpha1.MoldingKindSignoz: + containerNames, err := enricher.material.GetStringSlice("services|@keys") + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to get signoz container names") + } + + var apiServerAddr []string + var opampAddr []string + for _, containerName := range containerNames { + if strings.Contains(containerName, "-signoz") { + apiServerAddr = append(apiServerAddr, domain.MustNewAddress("tcp", containerName, 8080).String()) + opampAddr = append(opampAddr, domain.MustNewAddress("ws", containerName, 4320).String()) + } + } + config.Spec.Signoz.Status.Addresses.APIServer = apiServerAddr + config.Spec.Signoz.Status.Addresses.Opamp = opampAddr + + case v1alpha1.MoldingKindTelemetryKeeper: + containerNames, err := enricher.material.GetStringSlice("services|@keys") + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to get telemetrykeeper container names") + } + + var telemetrykeeperContainerNames []string + for _, containerName := range containerNames { + if strings.Contains(containerName, "telemetrykeeper") { + telemetrykeeperContainerNames = append(telemetrykeeperContainerNames, domain.MustNewAddress("tcp", containerName, 9181).String()) + } + } + config.Spec.TelemetryKeeper.Status.Addresses.Client = telemetrykeeperContainerNames + + var telemetryRaftaddress []string + for _, containerName := range containerNames { + if strings.Contains(containerName, "telemetrykeeper") { + telemetryRaftaddress = append(telemetryRaftaddress, domain.MustNewAddress("tcp", containerName, 9234).String()) + } + } + config.Spec.TelemetryKeeper.Status.Addresses.Raft = telemetryRaftaddress + + case v1alpha1.MoldingKindMetaStore: + // Skip molding enrichment if sqlite + if config.Spec.MetaStore.Kind == installation.MetaStoreKindSQLite { + return nil + } + containerNames, err := enricher.material.GetStringSlice("services|@keys") + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to get metastore container names") + } + + var metastoreContainerNames []string + for _, containerName := range containerNames { + if strings.Contains(containerName, "metastore") { + metastoreContainerNames = append(metastoreContainerNames, domain.MustNewAddress("tcp", containerName, 5432).String()) + } + } + config.Spec.MetaStore.Status.Addresses.DSN = metastoreContainerNames + + case v1alpha1.MoldingKindIngester: + // The ingester is scaled via `deploy.replicas` and reached through the + // `<metadata.name>-ingester` network alias on the default network, + // which compose load-balances across all replicas. + config.Spec.Ingester.Status.Addresses.OTLP = []string{ + domain.MustNewAddress("tcp", config.Metadata.Name+"-ingester", 4318).String(), + domain.MustNewAddress("tcp", config.Metadata.Name+"-ingester", 4317).String(), + + } + } + + return nil +} diff --git a/internal/casting/coolifycasting/templates/coolify.yaml.gotmpl b/internal/casting/coolifycasting/templates/coolify.yaml.gotmpl new file mode 100644 index 0000000..9790a22 --- /dev/null +++ b/internal/casting/coolifycasting/templates/coolify.yaml.gotmpl @@ -0,0 +1,286 @@ +# documentation: https://signoz.io/docs/introduction/ +# slogan: An observability platform native to OpenTelemetry with logs, traces and metrics. +# tags: telemetry, server, applications, interface, logs, monitoring, traces, metrics +# logo: svgs/signoz.svg +# port: 8080 + +services: + {{- range $replicaIdx := until (int $.Spec.TelemetryKeeper.Spec.Cluster.Replicas) }} + {{ $.Metadata.Name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-{{ $replicaIdx }}: + image: {{ $.Spec.TelemetryKeeper.Spec.Image }} + healthcheck: + test: + - CMD + - clickhouse-keeper-client + - -h + - localhost + - -p + - "9181" + - -q + - ls + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + volumes: + - type: volume + source: {{ $.Metadata.Name }}-telemetrykeeper-{{ $replicaIdx }}-data + target: /var/lib/clickhouse-keeper + - type: bind + source: ./configs/telemetrykeeper/keeper-{{ $replicaIdx }}.yaml + target: /etc/clickhouse-keeper/keeper.yaml + content: | +{{ index $.Spec.TelemetryKeeper.Spec.Config.Data (printf "keeper-%d.yaml" $replicaIdx) | indent 10 }} + entrypoint: + - /usr/bin/clickhouse-keeper + - --config-file=/etc/clickhouse-keeper/keeper.yaml + logging: + options: + max-size: 50m + max-file: "3" + {{- end }} + {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-user-scripts: + image: {{ $.Spec.TelemetryStore.Spec.Image }} + restart: "no" + command: + - bash + - -c + - | + version="v0.0.1" + node_os=$$(uname -s | tr '[:upper:]' '[:lower:]') + node_arch=$$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) + echo "Fetching histogram-binary for $${node_os}/$${node_arch}" + cd /tmp + wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F$${version}/histogram-quantile_$${node_os}_$${node_arch}.tar.gz" + tar -xvzf histogram-quantile.tar.gz + mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile + exclude_from_hc: true + volumes: + - type: volume + source: {{ $.Metadata.Name }}-telemetrystore-user-scripts + target: /var/lib/clickhouse/user_scripts + logging: + options: + max-size: 50m + max-file: "3" + {{- range $shardIdx := .Spec.TelemetryStore.Spec.Cluster.Shards }} + {{- range $replicaIdx := add $.Spec.TelemetryStore.Spec.Cluster.Replicas 1 }} + {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-{{ $shardIdx }}-{{ $replicaIdx }}: + image: {{ $.Spec.TelemetryStore.Spec.Image }} + depends_on: + {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-user-scripts: + condition: service_completed_successfully + {{- range $keeperIdx := until (int $.Spec.TelemetryKeeper.Spec.Cluster.Replicas) }} + {{ $.Metadata.Name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-{{ $keeperIdx }}: + condition: service_healthy + {{- end }} + environment: + - CLICKHOUSE_SKIP_USER_SETUP=1 + - CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.yaml + healthcheck: + test: + - CMD + - wget + - --spider + - -q + - 0.0.0.0:8123/ping + interval: 30s + timeout: 5s + retries: 3 + volumes: + - type: volume + source: {{ $.Metadata.Name }}-telemetrystore-{{ $shardIdx }}-{{ $replicaIdx }}-data + target: /var/lib/clickhouse/ + - type: volume + source: {{ $.Metadata.Name }}-telemetrystore-user-scripts + target: /var/lib/clickhouse/user_scripts + read_only: true + - type: bind + source: ./configs/telemetrystore/config-{{ $shardIdx }}-{{ $replicaIdx }}.yaml + target: /etc/clickhouse-server/config.yaml + content: | +{{ index $.Spec.TelemetryStore.Spec.Config.Data (printf "config-%d-%d.yaml" $shardIdx $replicaIdx) | indent 10 }} + - type: bind + source: ./configs/telemetrystore/functions.yaml + target: /etc/clickhouse-server/functions.yaml + content: | +{{ index $.Spec.TelemetryStore.Spec.Config.Data "functions.yaml" | indent 10 }} + logging: + options: + max-size: 50m + max-file: "3" + {{- end }} + {{- end }} + {{- if ne $.Spec.MetaStore.Kind.String "sqlite" }} + {{- range $replicaIdx := until (int $.Spec.MetaStore.Spec.Cluster.Replicas) }} + {{ $.Metadata.Name }}-metastore-{{ $.Spec.MetaStore.Kind }}-{{ $replicaIdx }}: + image: {{ $.Spec.MetaStore.Spec.Image }} + environment: + - POSTGRES_USER=signoz + - POSTGRES_PASSWORD=signoz + - POSTGRES_DB=signoz + healthcheck: + test: + - CMD-SHELL + - pg_isready -U signoz -d signoz + interval: 30s + timeout: 5s + retries: 3 + volumes: + - type: volume + source: {{ $.Metadata.Name }}-metastore-{{ $replicaIdx }}-data + target: /var/lib/postgresql/data + logging: + options: + max-size: 50m + max-file: "3" + {{- end }} + {{- end }} + {{- range $replicaIdx := until (int $.Spec.Signoz.Spec.Cluster.Replicas) }} + {{ $.Metadata.Name }}-signoz-{{ $replicaIdx }}: + container_name: {{ $.Metadata.Name }}-signoz-{{ $replicaIdx }} + image: {{ $.Spec.Signoz.Spec.Image }} + depends_on: + {{- if ne $.Spec.MetaStore.Kind.String "sqlite" }} + {{- range $metaIdx := until (int $.Spec.MetaStore.Spec.Cluster.Replicas) }} + {{ $.Metadata.Name }}-metastore-{{ $.Spec.MetaStore.Kind }}-{{ $metaIdx }}: + condition: service_healthy + {{- end }} + {{- end }} + {{- range $shardIdx := $.Spec.TelemetryStore.Spec.Cluster.Shards }} + {{- range $storeReplicaIdx := add $.Spec.TelemetryStore.Spec.Cluster.Replicas 1 }} + {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-{{ $shardIdx }}-{{ $storeReplicaIdx }}: + condition: service_healthy + {{- end }} + {{- end }} + {{ $.Metadata.Name }}-telemetrystore-migrator: + condition: service_completed_successfully + healthcheck: + test: + - CMD + - wget + - --spider + - -q + - localhost:8080/api/v1/health + interval: 30s + timeout: 5s + retries: 3 + environment: + {{- if $.Spec.Signoz.Spec.Env }} + {{- range $key, $value := $.Spec.Signoz.Spec.Env }} + - {{ $key }}={{ $value }} + {{- end }} + {{- end }} + volumes: + - type: volume + source: {{ $.Metadata.Name }}-signoz-{{ $replicaIdx }}-data + target: /var/lib/signoz/ + logging: + options: + max-size: 50m + max-file: "3" + {{- end }} + {{- $ingesterReplicas := int $.Spec.Ingester.Spec.Cluster.Replicas }} + ingester: + image: {{ $.Spec.Ingester.Spec.Image }} + networks: + default: + aliases: + - {{ $.Metadata.Name }}-ingester + depends_on: + {{- range $shardIdx := $.Spec.TelemetryStore.Spec.Cluster.Shards }} + {{- range $storeReplicaIdx := add $.Spec.TelemetryStore.Spec.Cluster.Replicas 1 }} + {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-{{ $shardIdx }}-{{ $storeReplicaIdx }}: + condition: service_healthy + {{- end }} + {{- end }} + {{ $.Metadata.Name }}-telemetrystore-migrator: + condition: service_completed_successfully + {{- range $signozReplicaIdx := until (int $.Spec.Signoz.Spec.Cluster.Replicas) }} + {{ $.Metadata.Name }}-signoz-{{ $signozReplicaIdx }}: + condition: service_started + {{- end }} + environment: + - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux + - LOW_CARDINAL_EXCEPTION_GROUPING=false + {{- if $.Spec.Ingester.Spec.Env }} + {{- range $key, $value := $.Spec.Ingester.Spec.Env }} + - {{ $key }}={{ $value }} + {{- end }} + {{- end }} + entrypoint: + - /bin/sh + command: + - -c + - | + /signoz-otel-collector migrate sync check && + /signoz-otel-collector --config=/etc/otel-collector-config.yaml --manager-config=/etc/opamp-config.yaml --copy-path=/var/tmp/collector-config.yaml + volumes: + - type: bind + source: ./configs/ingester/ingester.yaml + target: /etc/otel-collector-config.yaml + content: | +{{ index $.Spec.Ingester.Spec.Config.Data "ingester.yaml" | indent 10 }} + - type: bind + source: ./configs/ingester/opamp.yaml + target: /etc/opamp-config.yaml + content: | +{{ index $.Spec.Ingester.Spec.Config.Data "opamp.yaml" | indent 10 }} + logging: + options: + max-size: 50m + max-file: "3" + deploy: + replicas: {{ $ingesterReplicas }} + {{ $.Metadata.Name }}-telemetrystore-migrator: + image: {{ $.Spec.Ingester.Spec.Image }} + depends_on: + {{- range $shardIdx := $.Spec.TelemetryStore.Spec.Cluster.Shards }} + {{- range $replicaIdx := add $.Spec.TelemetryStore.Spec.Cluster.Replicas 1 }} + {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-{{ $shardIdx }}-{{ $replicaIdx }}: + condition: service_healthy + {{- end }} + {{- end }} + {{- if $.Spec.Ingester.Spec.Env }} + environment: + {{- range $key, $value := $.Spec.Ingester.Spec.Env }} + - {{ $key }}={{ $value }} + {{- end }} + {{- end }} + restart: on-failure + exclude_from_hc: true + {{- $clickhouseDSN := printf "%s-telemetrystore-%s-0-0" $.Metadata.Name $.Spec.TelemetryStore.Kind }} + entrypoint: + - "/bin/sh" + - "-c" + command: + - /signoz-otel-collector migrate bootstrap --clickhouse-dsn tcp://{{ $clickhouseDSN }}:9000 + - /signoz-otel-collector migrate sync up --clickhouse-dsn tcp://{{ $clickhouseDSN }}:9000 + - /signoz-otel-collector migrate async up --clickhouse-dsn tcp://{{$clickhouseDSN }}:9000 + logging: + options: + max-size: 50m + max-file: "3" +volumes: + {{- range $replicaIdx := until (int $.Spec.TelemetryKeeper.Spec.Cluster.Replicas) }} + {{ $.Metadata.Name }}-telemetrykeeper-{{ $replicaIdx }}-data: + name: {{ $.Metadata.Name }}-telemetrykeeper-{{ $replicaIdx }}-data + {{- end }} + {{- range $shardIdx := $.Spec.TelemetryStore.Spec.Cluster.Shards }} + {{- range $replicaIdx := add $.Spec.TelemetryStore.Spec.Cluster.Replicas 1 }} + {{ $.Metadata.Name }}-telemetrystore-{{ $shardIdx }}-{{ $replicaIdx }}-data: + name: {{ $.Metadata.Name }}-telemetrystore-{{ $shardIdx }}-{{ $replicaIdx }}-data + {{- end }} + {{- end }} + {{ $.Metadata.Name }}-telemetrystore-user-scripts: + name: {{ $.Metadata.Name }}-telemetrystore-user-scripts + {{- if ne $.Spec.MetaStore.Kind.String "sqlite" }} + {{- range $replicaIdx := until (int $.Spec.MetaStore.Spec.Cluster.Replicas) }} + {{ $.Metadata.Name }}-metastore-{{ $replicaIdx }}-data: + name: {{ $.Metadata.Name }}-metastore-{{ $replicaIdx }}-data + {{- end }} + {{- end }} + {{- range $replicaIdx := until (int $.Spec.Signoz.Spec.Cluster.Replicas) }} + {{ $.Metadata.Name }}-signoz-{{ $replicaIdx }}-data: + name: {{ $.Metadata.Name }}-signoz-{{ $replicaIdx }}-data + {{- end }} diff --git a/internal/casting/dockercomposecasting/casting.go b/internal/casting/dockercomposecasting/casting.go index 79b0200..b3acc4d 100644 --- a/internal/casting/dockercomposecasting/casting.go +++ b/internal/casting/dockercomposecasting/casting.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "errors" - "fmt" "log/slog" "os" "os/exec" @@ -22,59 +21,59 @@ var _ rootcasting.Casting = (*dockerComposeCasting)(nil) type dockerComposeCasting struct { logger *slog.Logger - castings []*types.Template + castings []*domain.Template } func New(logger *slog.Logger) *dockerComposeCasting { return &dockerComposeCasting{ logger: logger, - castings: []*types.Template{ + castings: []*domain.Template{ composeYAMLTemplate, }, } } -func (casting *dockerComposeCasting) Enricher(ctx context.Context, config *v1alpha1.Casting) (molding.MoldingEnricher, error) { +func (casting *dockerComposeCasting) Enricher(ctx context.Context, config *installation.Casting) (molding.MoldingEnricher, error) { return newDockerComposeMoldingEnricher(config) } -func (casting *dockerComposeCasting) Forge(ctx context.Context, config v1alpha1.Casting, poursPath string) ([]types.Material, error) { +func (casting *dockerComposeCasting) Forge(ctx context.Context, config installation.Casting, poursPath string) ([]domain.Material, error) { buf := bytes.NewBuffer(nil) err := composeYAMLTemplate.Execute(buf, config) if err != nil { - return nil, fmt.Errorf("failed to execute compose yaml template: %w", err) + return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to execute compose yaml template") } - composeMaterial, err := types.NewYAMLMaterial(buf.Bytes(), filepath.Join(rootcasting.DeploymentDir, "compose.yaml")) + composeMaterial, err := domain.NewYAMLMaterial(buf.Bytes(), filepath.Join(rootcasting.DeploymentDir, "compose.yaml")) if err != nil { - return nil, fmt.Errorf("failed to create compose yaml material: %w", err) + return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to create compose yaml material") } - materials := []types.Material{composeMaterial} + materials := []domain.Material{composeMaterial} // Add telemetrykeeper config files for filename, content := range config.Spec.TelemetryKeeper.Spec.Config.Data { - material, err := types.NewYAMLMaterial([]byte(content), filepath.Join(rootcasting.DeploymentDir, "configs", "telemetrykeeper", filename)) + material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(rootcasting.DeploymentDir, "telemetrykeeper", config.Spec.TelemetryKeeper.Kind.String(), filename)) if err != nil { - return nil, fmt.Errorf("failed to create telemetrykeeper config material: %w", err) + return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to create telemetrykeeper config material") } materials = append(materials, material) } // Add telemetrystore config files for filename, content := range config.Spec.TelemetryStore.Spec.Config.Data { - material, err := types.NewYAMLMaterial([]byte(content), filepath.Join(rootcasting.DeploymentDir, "configs", "telemetrystore", filename)) + material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(rootcasting.DeploymentDir, "telemetrystore", config.Spec.TelemetryStore.Kind.String(), filename)) if err != nil { - return nil, fmt.Errorf("failed to create telemetrystore config material: %w", err) + return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to create telemetrystore config material") } materials = append(materials, material) } // Add metastore config files for filename, content := range config.Spec.MetaStore.Spec.Config.Data { - material, err := types.NewYAMLMaterial([]byte(content), filepath.Join(rootcasting.DeploymentDir, "configs", "metastore", filename)) + material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(rootcasting.DeploymentDir, "metastore", config.Spec.MetaStore.Kind.String(), filename)) if err != nil { - return nil, fmt.Errorf("failed to create metastore config material: %w", err) + return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to create metastore config material") } materials = append(materials, material) } @@ -90,9 +89,9 @@ func (casting *dockerComposeCasting) Forge(ctx context.Context, config v1alpha1. // Add ingester config files for filename, content := range config.Spec.Ingester.Spec.Config.Data { - material, err := types.NewYAMLMaterial([]byte(content), filepath.Join(rootcasting.DeploymentDir, "configs", "ingester", filename)) + material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(rootcasting.DeploymentDir, "ingester", filename)) if err != nil { - return nil, fmt.Errorf("failed to create ingester config material: %w", err) + return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to create ingester config material") } materials = append(materials, material) } @@ -100,13 +99,13 @@ func (casting *dockerComposeCasting) Forge(ctx context.Context, config v1alpha1. return materials, nil } -func (casting *dockerComposeCasting) Cast(ctx context.Context, config v1alpha1.Casting, outputPath string) error { +func (casting *dockerComposeCasting) Cast(ctx context.Context, config installation.Casting, outputPath string) error { casting.logger.InfoContext(ctx, "Executing commands for platform") // Check if compose file exists composeFile := filepath.Join(outputPath, rootcasting.DeploymentDir, "compose.yaml") if _, err := os.Stat(composeFile); os.IsNotExist(err) { - return fmt.Errorf("compose file does not exist at path: %s", composeFile) + return foundryerrors.Newf(foundryerrors.TypeNotFound, "compose file does not exist at path: %s", composeFile) } // Create a context with 5-minute timeout @@ -117,11 +116,10 @@ func (casting *dockerComposeCasting) Cast(ctx context.Context, config v1alpha1.C composeCmd, err := getComposeCommand(runctx) if err != nil { casting.logger.ErrorContext(runctx, "Docker compose not available", slog.String("error", err.Error())) - return fmt.Errorf("docker compose not available: %w", err) + return foundryerrors.Wrapf(err, foundryerrors.TypeNotFound, "docker compose not available") } - // Build command arguments: "up -d" - args := append(composeCmd[1:], "-p", config.Metadata.Name, "-f", composeFile, "up", "-d") + args := append(composeCmd[1:], "-f", composeFile, "up", "-d") casting.logger.DebugContext(runctx, "Running command", slog.String("command", strings.Join(append([]string{composeCmd[0]}, args...), " "))) @@ -140,14 +138,14 @@ func (casting *dockerComposeCasting) Cast(ctx context.Context, config v1alpha1.C return nil } -func getComposeMaterial(config *v1alpha1.Casting, path string) (types.Material, error) { +func getComposeMaterial(config *installation.Casting, path string) (domain.StructuredMaterial, error) { buf := bytes.NewBuffer(nil) err := composeYAMLTemplate.Execute(buf, config) if err != nil { - return types.Material{}, fmt.Errorf("failed to execute compose yaml template: %w", err) + return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to execute compose yaml template") } - return types.NewYAMLMaterial(buf.Bytes(), path) + return domain.NewYAMLMaterial(buf.Bytes(), path) } // getComposeCommand detects the available docker compose command. diff --git a/internal/casting/dockercomposecasting/embed.go b/internal/casting/dockercomposecasting/embed.go index a6fce59..1f07e59 100644 --- a/internal/casting/dockercomposecasting/embed.go +++ b/internal/casting/dockercomposecasting/embed.go @@ -10,5 +10,5 @@ import ( var templates embed.FS var ( - composeYAMLTemplate *types.Template = types.MustNewTemplateFromFS(templates, "templates/compose.yaml.gotmpl", types.FormatYAML) + composeYAMLTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/compose.yaml.gotmpl", domain.FormatYAML) ) diff --git a/internal/casting/dockercomposecasting/enricher.go b/internal/casting/dockercomposecasting/enricher.go index 9891d57..4f53914 100644 --- a/internal/casting/dockercomposecasting/enricher.go +++ b/internal/casting/dockercomposecasting/enricher.go @@ -2,7 +2,6 @@ package dockercomposecasting import ( "context" - "fmt" "path/filepath" "strings" @@ -15,31 +14,31 @@ import ( var _ molding.MoldingEnricher = (*dockerComposeMoldingEnricher)(nil) type dockerComposeMoldingEnricher struct { - material types.Material + material domain.StructuredMaterial } -func newDockerComposeMoldingEnricher(config *v1alpha1.Casting) (*dockerComposeMoldingEnricher, error) { +func newDockerComposeMoldingEnricher(config *installation.Casting) (*dockerComposeMoldingEnricher, error) { material, err := getComposeMaterial(config, filepath.Join(rootcasting.DeploymentDir, "compose.yaml")) if err != nil { - return nil, fmt.Errorf("failed to get compose yaml material: %w", err) + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to get compose yaml material") } return &dockerComposeMoldingEnricher{material: material}, nil } -func (enricher *dockerComposeMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *v1alpha1.Casting) error { +func (enricher *dockerComposeMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *installation.Casting) error { switch kind { case v1alpha1.MoldingKindTelemetryStore: // Get telemetrystore container names containerNames, err := enricher.material.GetStringSlice("services|@keys") if err != nil { - return fmt.Errorf("failed to get telemetrystore container names: %w", err) + return errors.Wrapf(err, errors.TypeInternal, "failed to get telemetrystore container names") } var telemetrystoreContainerNames []string for _, containerName := range containerNames { if strings.Contains(containerName, "telemetrystore-clickhouse") && !strings.Contains(containerName, "user-scripts") { - telemetrystoreContainerNames = append(telemetrystoreContainerNames, types.FormatAddress("tcp", containerName, 9000)) + telemetrystoreContainerNames = append(telemetrystoreContainerNames, domain.MustNewAddress("tcp", containerName, 9000).String()) } } @@ -67,13 +66,13 @@ func (enricher *dockerComposeMoldingEnricher) EnrichStatus(ctx context.Context, // Get telemetrykeeper container names (using service keys since they match container_name) containerNames, err := enricher.material.GetStringSlice("services|@keys") if err != nil { - return fmt.Errorf("failed to get telemetrykeeper container names: %w", err) + return errors.Wrapf(err, errors.TypeInternal, "failed to get telemetrykeeper container names") } var telemetrykeeperContainerNames []string for _, containerName := range containerNames { if strings.Contains(containerName, "telemetrykeeper") { - telemetrykeeperContainerNames = append(telemetrykeeperContainerNames, types.FormatAddress("tcp", containerName, 9181)) + telemetrykeeperContainerNames = append(telemetrykeeperContainerNames, domain.MustNewAddress("tcp", containerName, 9181).String()) } } @@ -82,43 +81,40 @@ func (enricher *dockerComposeMoldingEnricher) EnrichStatus(ctx context.Context, var telemetryRaftaddress []string for _, containerName := range containerNames { if strings.Contains(containerName, "telemetrykeeper") { - telemetryRaftaddress = append(telemetryRaftaddress, types.FormatAddress("tcp", containerName, 9234)) + telemetryRaftaddress = append(telemetryRaftaddress, domain.MustNewAddress("tcp", containerName, 9234).String()) } } config.Spec.TelemetryKeeper.Status.Addresses.Raft = telemetryRaftaddress case v1alpha1.MoldingKindMetaStore: + // Skip molding enrichment if sqlite + if config.Spec.MetaStore.Kind == installation.MetaStoreKindSQLite { + return nil + } // Get metastore container names containerNames, err := enricher.material.GetStringSlice("services|@keys") if err != nil { - return fmt.Errorf("failed to get metastore container names: %w", err) + return errors.Wrapf(err, errors.TypeInternal, "failed to get metastore container names") } var metastoreContainerNames []string for _, containerName := range containerNames { if strings.Contains(containerName, "metastore") { - metastoreContainerNames = append(metastoreContainerNames, types.FormatAddress("tcp", containerName, 9000)) + metastoreContainerNames = append(metastoreContainerNames, domain.MustNewAddress("tcp", containerName, 5432).String()) } } config.Spec.MetaStore.Status.Addresses.DSN = metastoreContainerNames case v1alpha1.MoldingKindIngester: - // Get ingester container names - containerNames, err := enricher.material.GetStringSlice("services|@keys") - if err != nil { - return fmt.Errorf("failed to get ingester container names: %w", err) + // The ingester is scaled via `deploy.replicas` and reached through the + // `<metadata.name>-ingester` network alias, which compose load-balances + // across all replicas. + config.Spec.Ingester.Status.Addresses.OTLP = []string{ + domain.MustNewAddress("tcp", config.Metadata.Name+"-ingester", 4318).String(), + domain.MustNewAddress("tcp", config.Metadata.Name+"-ingester", 4317).String(), } - - var ingesterContainerNames []string - for _, containerName := range containerNames { - if strings.Contains(containerName, "ingester") { - ingesterContainerNames = append(ingesterContainerNames, types.FormatAddress("tcp", containerName, 9000)) - } - } - - config.Spec.Ingester.Status.Addresses.OTLP = ingesterContainerNames } return nil diff --git a/internal/casting/dockercomposecasting/templates/compose.yaml.gotmpl b/internal/casting/dockercomposecasting/templates/compose.yaml.gotmpl index 4d38250..a05ed85 100644 --- a/internal/casting/dockercomposecasting/templates/compose.yaml.gotmpl +++ b/internal/casting/dockercomposecasting/templates/compose.yaml.gotmpl @@ -1,13 +1,20 @@ +name: {{ $.Metadata.Name }} services: {{- range $replicaIdx := until (int $.Spec.TelemetryKeeper.Spec.Cluster.Replicas) }} {{ $.Metadata.Name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-{{ $replicaIdx }}: - container_name: {{ $.Metadata.Name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-{{ $replicaIdx }} + container_name: {{ $.Metadata.Name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-{{ $replicaIdx }} image: {{ $.Spec.TelemetryKeeper.Spec.Image }} networks: - {{ $.Metadata.Name }}-network + {{- if $.Spec.TelemetryKeeper.Spec.Env }} + environment: + {{- range $key, $value := $.Spec.TelemetryKeeper.Spec.Env }} + - {{ $key }}={{ $value }} + {{- end }} + {{- end }} volumes: - {{ $.Metadata.Name }}-telemetrykeeper-{{ $replicaIdx }}-data:/var/lib/clickhouse-keeper - - ./configs/telemetrykeeper/keeper-{{ $replicaIdx }}.yaml:/etc/clickhouse-keeper/keeper.yaml:ro + - ./telemetrykeeper/{{ $.Spec.TelemetryKeeper.Kind }}/keeper-{{ $replicaIdx }}.yaml:/etc/clickhouse-keeper/keeper.yaml:ro entrypoint: - /usr/bin/clickhouse-keeper - --config-file=/etc/clickhouse-keeper/keeper.yaml @@ -26,8 +33,8 @@ services: retries: 3 start_period: 40s {{- end }} - {{- range $shardIdx := .Spec.TelemetryStore.Spec.Cluster.Shards }} - {{- range $replicaIdx := add $.Spec.TelemetryStore.Spec.Cluster.Replicas 1 }} + {{- range $shardIdx := until (int $.Spec.TelemetryStore.Spec.Cluster.Shards) }} + {{- range $replicaIdx := until (int (add 1 (int $.Spec.TelemetryStore.Spec.Cluster.Replicas))) }} {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-{{ $shardIdx }}-{{ $replicaIdx }}: container_name: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-{{ $shardIdx }}-{{ $replicaIdx }} depends_on: @@ -43,10 +50,16 @@ services: environment: - CLICKHOUSE_SKIP_USER_SETUP=1 - CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.yaml + {{- if $.Spec.TelemetryStore.Spec.Env }} + {{- range $key, $value := $.Spec.TelemetryStore.Spec.Env }} + - {{ $key }}={{ $value }} + {{- end }} + {{- end }} volumes: - {{ $.Metadata.Name }}-telemetrystore-{{ $shardIdx }}-{{ $replicaIdx }}-data:/var/lib/clickhouse - {{ $.Metadata.Name }}-telemetrystore-user-scripts:/var/lib/clickhouse/user_scripts:ro - - ./configs/telemetrystore/config.yaml:/etc/clickhouse-server/config.yaml:ro + - ./telemetrystore/{{ $.Spec.TelemetryStore.Kind }}/config-{{ $shardIdx }}-{{ $replicaIdx }}.yaml:/etc/clickhouse-server/config.yaml:ro + - ./telemetrystore/{{ $.Spec.TelemetryStore.Kind }}/functions.yaml:/etc/clickhouse-server/functions.yaml:ro healthcheck: test: - "CMD" @@ -79,18 +92,20 @@ services: wget -O histogram-quantile.tar.gz "https://github.com/Hanzo O11y/o11y/releases/download/histogram-quantile%2F$${version}/histogram-quantile_$${node_os}_$${node_arch}.tar.gz" tar -xvzf histogram-quantile.tar.gz mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile + {{- if ne $.Spec.MetaStore.Kind.String "sqlite" }} {{- range $replicaIdx := until (int $.Spec.MetaStore.Spec.Cluster.Replicas) }} {{ $.Metadata.Name }}-metastore-{{ $.Spec.MetaStore.Kind }}-{{ $replicaIdx }}: container_name: {{ $.Metadata.Name }}-metastore-{{ $.Spec.MetaStore.Kind }}-{{ $replicaIdx }} - image: docker.io/{{ $.Spec.MetaStore.Kind }}:{{ $.Spec.MetaStore.Spec.Version }} + image: {{ $.Spec.MetaStore.Spec.Image }} networks: - {{ $.Metadata.Name }}-network + {{- if $.Spec.MetaStore.Spec.Env }} environment: - POSTGRES_USER=o11y - POSTGRES_PASSWORD=o11y - POSTGRES_DB=o11y volumes: - - {{ $.Metadata.Name }}-metastore-{{ $replicaIdx }}-data:/var/lib/postgresql/data + - {{ $.Metadata.Name }}-metastore-{{ $.Spec.MetaStore.Kind }}-{{ $replicaIdx }}-data:/var/lib/postgresql/data healthcheck: test: - "CMD-SHELL" @@ -100,37 +115,42 @@ services: retries: 3 start_period: 30s {{- end }} - {{ $.Metadata.Name }}-ingester: + {{- end }} + {{- $ingesterReplicas := int $.Spec.Ingester.Spec.Cluster.Replicas }} + ingester: image: {{ $.Spec.Ingester.Spec.Image }} networks: - - {{ $.Metadata.Name }}-network + {{ $.Metadata.Name }}-network: + aliases: + - {{ $.Metadata.Name }}-ingester + entrypoint: + - /bin/sh command: - - "--config=/etc/otel-collector-config.yaml" + - -c + - | + /signoz-otel-collector migrate sync check && + /signoz-otel-collector --config=/etc/otel-collector-config.yaml --manager-config=/etc/opamp-config.yaml --copy-path=/var/tmp/collector-config.yaml volumes: - - ./configs/ingester/ingester.yaml:/etc/otel-collector-config.yaml:ro + - ./ingester/ingester.yaml:/etc/otel-collector-config.yaml:ro + - ./ingester/opamp.yaml:/etc/opamp-config.yaml:ro + {{- if eq $ingesterReplicas 1 }} ports: - - "4317:4317" # OTLP gRPC - - "4318:4318" # OTLP HTTP - healthcheck: - test: - - "CMD" - - "wget" - - "--spider" - - "-q" - - "http://localhost:13133/health" - interval: 30s - timeout: 10s - retries: 3 - start_period: 30s - depends_on: - {{- range $shardIdx := $.Spec.TelemetryStore.Spec.Cluster.Shards }} - {{- range $replicaIdx := add $.Spec.TelemetryStore.Spec.Cluster.Replicas 1 }} - {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-{{ $shardIdx }}-{{ $replicaIdx }}: - condition: service_healthy + - "4317:4317" + - "4318:4318" + {{- else }} + expose: + - "4317" + - "4318" + {{- end }} + environment: +{{ if and $.Spec.TelemetryStore.Spec.Enabled $.Spec.TelemetryStore.Status.Addresses.TCP (index $.Spec.TelemetryStore.Status.Addresses.TCP 0) }} + - SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN={{ index $.Spec.TelemetryStore.Status.Addresses.TCP 0 }} +{{ end }} + {{- if $.Spec.Ingester.Spec.Env }} + {{- range $key, $value := $.Spec.Ingester.Spec.Env }} + - {{ $key }}={{ $value }} {{- end }} - {{- end }} - {{ $.Metadata.Name }}-telemetrystore-migrator: - condition: service_completed_successfully + {{- end }} deploy: replicas: {{ int $.Spec.Ingester.Spec.Cluster.Replicas }} {{- range $replicaIdx := until (int $.Spec.O11y.Spec.Cluster.Replicas) }} @@ -138,17 +158,7 @@ services: image: {{ $.Spec.O11y.Spec.Image }} networks: - {{ $.Metadata.Name }}-network - depends_on: - {{- range $metaIdx := until (int $.Spec.MetaStore.Spec.Cluster.Replicas) }} - {{ $.Metadata.Name }}-metastore-{{ $.Spec.MetaStore.Kind }}-{{ $metaIdx }}: - condition: service_healthy - {{- end }} - {{- range $shardIdx := $.Spec.TelemetryStore.Spec.Cluster.Shards }} - {{- range $storeReplicaIdx := add $.Spec.TelemetryStore.Spec.Cluster.Replicas 1 }} - {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-{{ $shardIdx }}-{{ $storeReplicaIdx }}: - condition: service_healthy - {{- end }} - {{- end }} + {{- if $.Spec.Signoz.Spec.Env }} environment: - HANZO_ALERTMANAGER_PROVIDER=o11y - HANZO_TELEMETRYSTORE_CLICKHOUSE_DSN=tcp://{{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-0-0:9000 @@ -160,14 +170,18 @@ services: - DEPLOYMENT_TYPE=docker-standalone-amd - DOT_METRICS_ENABLED=true ports: - - "8080:8080" + - "{{ add 8080 (mul $replicaIdx 1000) }}:8080" + {{- if eq $.Spec.MetaStore.Kind.String "sqlite" }} + volumes: + - {{ $.Metadata.Name }}-metastore-{{ $.Spec.MetaStore.Kind }}-{{ $replicaIdx }}-data:{{ dir (index $.Spec.Signoz.Spec.Env "SIGNOZ_SQLSTORE_SQLITE_PATH") }} + {{- end }} healthcheck: test: - "CMD" - "wget" - "--spider" - "-q" - - "http://localhost:3301/api/v1/health" + - "http://localhost:8080/api/v1/health" interval: 30s timeout: 10s retries: 3 @@ -178,13 +192,12 @@ services: - {{ $.Metadata.Name }}-o11y-{{ $replicaIdx }}-data:/var/lib/o11y {{- end }} {{ $.Metadata.Name }}-telemetrystore-migrator: - image: {{ $.Spec.Ingester.Spec.Image }} container_name: {{ $.Metadata.Name }}-telemetrystore-migrator + image: {{ $.Spec.Ingester.Spec.Image }} networks: - {{ $.Metadata.Name }}-network entrypoint: - - "/bin/sh" - - "-c" + - /bin/sh command: - /o11y-otel-collector migrate bootstrap --clickhouse-dsn tcp://{{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-0-0:9000 - /o11y-otel-collector migrate sync up --clickhouse-dsn tcp://{{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-0-0:9000 @@ -196,21 +209,27 @@ services: condition: service_healthy {{- end }} {{- end }} + {{- end }} networks: {{ $.Metadata.Name }}-network: name: {{ $.Metadata.Name }}-network volumes: {{- range $replicaIdx := until (int $.Spec.TelemetryKeeper.Spec.Cluster.Replicas) }} - {{ $.Metadata.Name }}-telemetrykeeper-{{ $replicaIdx }}-data: + {{ $.Metadata.Name }}-telemetrykeeper-{{ $replicaIdx }}-data: + name: {{ $.Metadata.Name }}-telemetrykeeper-{{ $replicaIdx }}-data {{- end }} - {{- range $shardIdx := $.Spec.TelemetryStore.Spec.Cluster.Shards }} - {{- range $replicaIdx := add $.Spec.TelemetryStore.Spec.Cluster.Replicas 1 }} + {{- range $shardIdx := until (int $.Spec.TelemetryStore.Spec.Cluster.Shards) }} + {{- range $replicaIdx := until (int (add 1 (int $.Spec.TelemetryStore.Spec.Cluster.Replicas))) }} {{ $.Metadata.Name }}-telemetrystore-{{ $shardIdx }}-{{ $replicaIdx }}-data: + name: {{ $.Metadata.Name }}-telemetrystore-{{ $shardIdx }}-{{ $replicaIdx }}-data {{- end }} {{- end }} {{ $.Metadata.Name }}-telemetrystore-user-scripts: + name: {{ $.Metadata.Name }}-telemetrystore-user-scripts + {{- if ne $.Spec.MetaStore.Kind.String "sqlite" }} {{- range $replicaIdx := until (int $.Spec.MetaStore.Spec.Cluster.Replicas) }} - {{ $.Metadata.Name }}-metastore-{{ $replicaIdx }}-data: + {{ $.Metadata.Name }}-metastore-{{ $.Spec.MetaStore.Kind }}-{{ $replicaIdx }}-data: + name: {{ $.Metadata.Name }}-metastore-{{ $.Spec.MetaStore.Kind }}-{{ $replicaIdx }}-data {{- end }} {{- range $replicaIdx := until (int $.Spec.O11y.Spec.Cluster.Replicas) }} {{ $.Metadata.Name }}-o11y-{{ $replicaIdx }}-data: diff --git a/internal/casting/dockerswarmcasting/casting.go b/internal/casting/dockerswarmcasting/casting.go new file mode 100644 index 0000000..e9c8dc8 --- /dev/null +++ b/internal/casting/dockerswarmcasting/casting.go @@ -0,0 +1,138 @@ +package dockerswarmcasting + +import ( + "bytes" + "context" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/signoz/foundry/api/v1alpha1/installation" + rootcasting "github.com/signoz/foundry/internal/casting" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/molding" +) + +var _ rootcasting.Casting = (*dockerSwarmCasting)(nil) + +type dockerSwarmCasting struct { + logger *slog.Logger + castings []*domain.Template +} + +func New(logger *slog.Logger) *dockerSwarmCasting { + return &dockerSwarmCasting{ + logger: logger, + castings: []*domain.Template{ + composeYAMLTemplate, + }, + } +} + +func (casting *dockerSwarmCasting) Enricher(ctx context.Context, config *installation.Casting) (molding.MoldingEnricher, error) { + return newDockerSwarmMoldingEnricher(config) +} + +func (casting *dockerSwarmCasting) Forge(ctx context.Context, config installation.Casting, poursPath string) ([]domain.Material, error) { + + buf := bytes.NewBuffer(nil) + err := composeYAMLTemplate.Execute(buf, config) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to execute compose yaml template") + } + + composeMaterial, err := domain.NewYAMLMaterial(buf.Bytes(), filepath.Join(rootcasting.DeploymentDir, "compose.yaml")) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to create compose yaml material") + } + + materials := []domain.Material{composeMaterial} + + for filename, content := range config.Spec.TelemetryKeeper.Spec.Config.Data { + material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(rootcasting.DeploymentDir, "telemetrykeeper", config.Spec.TelemetryKeeper.Kind.String(), filename)) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to create telemetrykeeper config material") + } + materials = append(materials, material) + } + + for filename, content := range config.Spec.TelemetryStore.Spec.Config.Data { + material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(rootcasting.DeploymentDir, "telemetrystore", config.Spec.TelemetryStore.Kind.String(), filename)) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to create telemetrystore config material") + } + materials = append(materials, material) + } + + for filename, content := range config.Spec.MetaStore.Spec.Config.Data { + material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(rootcasting.DeploymentDir, "metastore", config.Spec.MetaStore.Kind.String(), filename)) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to create metastore config material") + } + materials = append(materials, material) + } + + for filename, content := range config.Spec.Signoz.Spec.Config.Data { + material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(rootcasting.DeploymentDir, "signoz", filename)) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to create signoz config material") + } + materials = append(materials, material) + } + + for filename, content := range config.Spec.Ingester.Spec.Config.Data { + material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(rootcasting.DeploymentDir, "ingester", filename)) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to create ingester config material") + } + materials = append(materials, material) + } + + return materials, nil +} + +func (casting *dockerSwarmCasting) Cast(ctx context.Context, config installation.Casting, outputPath string) error { + casting.logger.InfoContext(ctx, "Deploying stack to Docker Swarm") + + composeFile := filepath.Join(outputPath, rootcasting.DeploymentDir, "compose.yaml") + if _, err := os.Stat(composeFile); os.IsNotExist(err) { + return errors.Newf(errors.TypeNotFound, "compose file does not exist at path: %s", composeFile) + } + + runctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + args := []string{"stack", "deploy", "-d", "-c", composeFile} + + args = append(args, config.Metadata.Name) + + casting.logger.DebugContext(runctx, "Running command", slog.String("command", strings.Join(append([]string{"docker"}, args...), " "))) + + cmd := exec.CommandContext(runctx, "docker", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + if err != nil { + casting.logger.ErrorContext(runctx, "Stack deploy failed", slog.String("error", err.Error())) + return err + } + + casting.logger.InfoContext(runctx, "Stack deployed successfully") + + return nil +} + +func getComposeMaterial(config *installation.Casting, path string) (domain.StructuredMaterial, error) { + buf := bytes.NewBuffer(nil) + err := composeYAMLTemplate.Execute(buf, config) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to execute compose yaml template") + } + + return domain.NewYAMLMaterial(buf.Bytes(), path) +} diff --git a/internal/casting/dockerswarmcasting/embed.go b/internal/casting/dockerswarmcasting/embed.go new file mode 100644 index 0000000..862ca58 --- /dev/null +++ b/internal/casting/dockerswarmcasting/embed.go @@ -0,0 +1,14 @@ +package dockerswarmcasting + +import ( + "embed" + + "github.com/signoz/foundry/internal/domain" +) + +//go:embed templates/*.gotmpl +var templates embed.FS + +var ( + composeYAMLTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/compose.yaml.gotmpl", domain.FormatYAML) +) diff --git a/internal/casting/dockerswarmcasting/enricher.go b/internal/casting/dockerswarmcasting/enricher.go new file mode 100644 index 0000000..9e8488d --- /dev/null +++ b/internal/casting/dockerswarmcasting/enricher.go @@ -0,0 +1,112 @@ +package dockerswarmcasting + +import ( + "context" + "path/filepath" + "strings" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/installation" + rootcasting "github.com/signoz/foundry/internal/casting" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/molding" +) + +var _ molding.MoldingEnricher = (*dockerSwarmMoldingEnricher)(nil) + +type dockerSwarmMoldingEnricher struct { + material domain.StructuredMaterial +} + +func newDockerSwarmMoldingEnricher(config *installation.Casting) (*dockerSwarmMoldingEnricher, error) { + material, err := getComposeMaterial(config, filepath.Join(rootcasting.DeploymentDir, "compose.yaml")) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to get compose yaml material") + } + + return &dockerSwarmMoldingEnricher{material: material}, nil +} + +func (enricher *dockerSwarmMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *installation.Casting) error { + switch kind { + case v1alpha1.MoldingKindTelemetryStore: + containerNames, err := enricher.material.GetStringSlice("services|@keys") + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to get telemetrystore service names") + } + + var telemetrystoreAddresses []string + for _, name := range containerNames { + if strings.Contains(name, "telemetrystore-clickhouse") && !strings.Contains(name, "user-scripts") { + telemetrystoreAddresses = append(telemetrystoreAddresses, domain.MustNewAddress("tcp", name, 9000).String()) + } + } + + config.Spec.TelemetryStore.Status.Addresses.TCP = telemetrystoreAddresses + + case v1alpha1.MoldingKindSignoz: + containerNames, err := enricher.material.GetStringSlice("services|@keys") + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to get signoz service names") + } + + var apiServerAddr []string + var opampAddr []string + for _, name := range containerNames { + if strings.Contains(name, "-signoz") { + apiServerAddr = append(apiServerAddr, domain.MustNewAddress("tcp", name, 8080).String()) + opampAddr = append(opampAddr, domain.MustNewAddress("ws", name, 4320).String()) + } + } + config.Spec.Signoz.Status.Addresses.APIServer = apiServerAddr + config.Spec.Signoz.Status.Addresses.Opamp = opampAddr + + case v1alpha1.MoldingKindTelemetryKeeper: + containerNames, err := enricher.material.GetStringSlice("services|@keys") + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to get telemetrykeeper service names") + } + + var clientAddresses []string + for _, name := range containerNames { + if strings.Contains(name, "telemetrykeeper") { + clientAddresses = append(clientAddresses, domain.MustNewAddress("tcp", name, 9181).String()) + } + } + config.Spec.TelemetryKeeper.Status.Addresses.Client = clientAddresses + + var raftAddresses []string + for _, name := range containerNames { + if strings.Contains(name, "telemetrykeeper") { + raftAddresses = append(raftAddresses, domain.MustNewAddress("tcp", name, 9234).String()) + } + } + config.Spec.TelemetryKeeper.Status.Addresses.Raft = raftAddresses + + case v1alpha1.MoldingKindMetaStore: + containerNames, err := enricher.material.GetStringSlice("services|@keys") + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to get metastore service names") + } + + var metastoreAddresses []string + for _, name := range containerNames { + if strings.Contains(name, "metastore") { + metastoreAddresses = append(metastoreAddresses, domain.MustNewAddress("tcp", name, 5432).String()) + } + } + config.Spec.MetaStore.Status.Addresses.DSN = metastoreAddresses + + case v1alpha1.MoldingKindIngester: + // The ingester is scaled via `deploy.replicas` and reached through the + // `<metadata.name>-ingester` network alias, which Swarm's routing mesh + // load-balances across all replicas. + config.Spec.Ingester.Status.Addresses.OTLP = []string{ + domain.MustNewAddress("tcp", config.Metadata.Name+"-ingester", 4318).String(), + domain.MustNewAddress("tcp", config.Metadata.Name+"-ingester", 4317).String(), + } + } + + return nil +} diff --git a/internal/casting/dockerswarmcasting/templates/compose.yaml.gotmpl b/internal/casting/dockerswarmcasting/templates/compose.yaml.gotmpl new file mode 100644 index 0000000..2640ecc --- /dev/null +++ b/internal/casting/dockerswarmcasting/templates/compose.yaml.gotmpl @@ -0,0 +1,261 @@ +version: "3" +services: + {{- range $replicaIdx := until (int $.Spec.TelemetryKeeper.Spec.Cluster.Replicas) }} + {{ $.Metadata.Name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-{{ $replicaIdx }}: + image: {{ $.Spec.TelemetryKeeper.Spec.Image }} + networks: + - {{ $.Metadata.Name }}-network + {{- if $.Spec.TelemetryKeeper.Spec.Env }} + environment: + {{- range $key, $value := $.Spec.TelemetryKeeper.Spec.Env }} + - {{ $key }}={{ $value }} + {{- end }} + {{- end }} + volumes: + - {{ $.Metadata.Name }}-telemetrykeeper-{{ $replicaIdx }}-data:/var/lib/clickhouse-keeper + configs: + - source: telemetrykeeper-{{ $replicaIdx }} + target: /etc/clickhouse-keeper/keeper.yaml + entrypoint: + - /usr/bin/clickhouse-keeper + - --config-file=/etc/clickhouse-keeper/keeper.yaml + deploy: + restart_policy: + condition: on-failure + healthcheck: + test: + - "CMD" + - "clickhouse-keeper-client" + - "-h" + - "localhost" + - "-p" + - "9181" + - "-q" + - "ls" + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + {{- end }} + {{- range $shardIdx := until (int .Spec.TelemetryStore.Spec.Cluster.Shards) }} + {{- range $replicaIdx := until (int (add1 (int $.Spec.TelemetryStore.Spec.Cluster.Replicas))) }} + {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-{{ $shardIdx }}-{{ $replicaIdx }}: + image: {{ $.Spec.TelemetryStore.Spec.Image }} + hostname: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-{{ $shardIdx }}-{{ $replicaIdx }} + networks: + - {{ $.Metadata.Name }}-network + environment: + - CLICKHOUSE_SKIP_USER_SETUP=1 + - CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.yaml + {{- if $.Spec.TelemetryStore.Spec.Env }} + {{- range $key, $value := $.Spec.TelemetryStore.Spec.Env }} + - {{ $key }}={{ $value }} + {{- end }} + {{- end }} + volumes: + - {{ $.Metadata.Name }}-telemetrystore-{{ $shardIdx }}-{{ $replicaIdx }}-data:/var/lib/clickhouse + - {{ $.Metadata.Name }}-telemetrystore-user-scripts:/var/lib/clickhouse/user_scripts:ro + configs: + - source: telemetrystore-config-{{ $shardIdx }}-{{ $replicaIdx }} + target: /etc/clickhouse-server/config.yaml + - source: telemetrystore-functions + target: /etc/clickhouse-server/functions.yaml + deploy: + restart_policy: + condition: on-failure + healthcheck: + test: + - "CMD" + - "wget" + - "--spider" + - "-q" + - "http://localhost:8123/ping" + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + {{- end }} + {{- end }} + {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-user-scripts: + image: {{ $.Spec.TelemetryStore.Spec.Image }} + networks: + - {{ $.Metadata.Name }}-network + volumes: + - {{ $.Metadata.Name }}-telemetrystore-user-scripts:/var/lib/clickhouse/user_scripts + command: + - bash + - -c + - |- + version="v0.0.1" + node_os=$$(uname -s | tr '[:upper:]' '[:lower:]') + node_arch=$$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) + echo "Fetching histogram-binary for $${node_os}/$${node_arch}" + cd /tmp + wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F$${version}/histogram-quantile_$${node_os}_$${node_arch}.tar.gz" + tar -xvzf histogram-quantile.tar.gz + mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile + deploy: + mode: global + restart_policy: + condition: none + {{- if ne $.Spec.MetaStore.Kind.String "sqlite" }} + {{- range $replicaIdx := until (int $.Spec.MetaStore.Spec.Cluster.Replicas) }} + {{ $.Metadata.Name }}-metastore-{{ $.Spec.MetaStore.Kind }}-{{ $replicaIdx }}: + image: {{ $.Spec.MetaStore.Spec.Image }} + networks: + - {{ $.Metadata.Name }}-network + {{- if $.Spec.MetaStore.Spec.Env }} + environment: + {{- range $key, $value := $.Spec.MetaStore.Spec.Env }} + - {{ $key }}={{ $value }} + {{- end }} + {{- end }} + volumes: + - {{ $.Metadata.Name }}-metastore-{{ $.Spec.MetaStore.Kind }}-{{ $replicaIdx }}-data:/var/lib/postgresql/data + deploy: + restart_policy: + condition: on-failure + healthcheck: + test: + - "CMD-SHELL" + - "pg_isready -U signoz -d signoz" + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + {{- end }} + {{- end }} + {{- $ingesterReplicas := int $.Spec.Ingester.Spec.Cluster.Replicas }} + ingester: + image: {{ $.Spec.Ingester.Spec.Image }} + networks: + {{ $.Metadata.Name }}-network: + aliases: + - {{ $.Metadata.Name }}-ingester + entrypoint: + - /bin/sh + command: + - -c + - | + /signoz-otel-collector migrate sync check && + /signoz-otel-collector --config=/etc/otel-collector-config.yaml --manager-config=/etc/opamp-config.yaml --copy-path=/var/tmp/collector-config.yaml + configs: + - source: ingester-config + target: /etc/otel-collector-config.yaml + - source: manager-config + target: /etc/opamp-config.yaml + ports: + - "4317:4317" + - "4318:4318" + environment: +{{ if and $.Spec.TelemetryStore.Spec.Enabled $.Spec.TelemetryStore.Status.Addresses.TCP (index $.Spec.TelemetryStore.Status.Addresses.TCP 0) }} + - SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN={{ index $.Spec.TelemetryStore.Status.Addresses.TCP 0 }} +{{ end }} + {{- if $.Spec.Ingester.Spec.Env }} + {{- range $key, $value := $.Spec.Ingester.Spec.Env }} + - {{ $key }}={{ $value }} + {{- end }} + {{- end }} + deploy: + replicas: {{ $ingesterReplicas }} + {{- range $replicaIdx := until (int $.Spec.Signoz.Spec.Cluster.Replicas) }} + {{ $.Metadata.Name }}-signoz-{{ $replicaIdx }}: + image: {{ $.Spec.Signoz.Spec.Image }} + networks: + - {{ $.Metadata.Name }}-network + {{- if $.Spec.Signoz.Spec.Env }} + environment: + {{- range $key, $value := $.Spec.Signoz.Spec.Env }} + - {{ $key }}={{ $value }} + {{- end }} + {{- end }} + ports: + - "{{ add 8080 (mul $replicaIdx 1000) }}:8080" + {{- if eq $.Spec.MetaStore.Kind.String "sqlite" }} + volumes: + - {{ $.Metadata.Name }}-metastore-{{ $.Spec.MetaStore.Kind }}-{{ $replicaIdx }}-data:{{ dir (index $.Spec.Signoz.Spec.Env "SIGNOZ_SQLSTORE_SQLITE_PATH") }} + {{- end }} + deploy: + restart_policy: + condition: on-failure + healthcheck: + test: + - "CMD" + - "wget" + - "--spider" + - "-q" + - "http://localhost:8080/api/v1/health" + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + {{- end }} + {{ $.Metadata.Name }}-telemetrystore-migrator: + image: {{ $.Spec.Ingester.Spec.Image }} + networks: + - {{ $.Metadata.Name }}-network + entrypoint: + - /bin/sh + command: + - -c + - | + /signoz-otel-collector migrate ready && + /signoz-otel-collector migrate bootstrap && + /signoz-otel-collector migrate sync up && + /signoz-otel-collector migrate async up + environment: +{{ if and $.Spec.TelemetryStore.Spec.Enabled $.Spec.TelemetryStore.Status.Addresses.TCP (index $.Spec.TelemetryStore.Status.Addresses.TCP 0) }} + - SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN={{ index $.Spec.TelemetryStore.Status.Addresses.TCP 0 }} +{{ end }} + {{- if $.Spec.Ingester.Spec.Env }} + {{- range $key, $value := $.Spec.Ingester.Spec.Env }} + - {{ $key }}={{ $value }} + {{- end }} + {{- end }} + deploy: + replicas: 1 + restart_policy: + condition: on-failure + max_attempts: 3 +networks: + {{ $.Metadata.Name }}-network: + name: {{ $.Metadata.Name }}-network + driver: overlay + attachable: true +volumes: + {{- range $replicaIdx := until (int $.Spec.TelemetryKeeper.Spec.Cluster.Replicas) }} + {{ $.Metadata.Name }}-telemetrykeeper-{{ $replicaIdx }}-data: + {{- end }} + {{- range $shardIdx := until (int $.Spec.TelemetryStore.Spec.Cluster.Shards) }} + {{- range $replicaIdx := until (int (add1 (int $.Spec.TelemetryStore.Spec.Cluster.Replicas))) }} + {{ $.Metadata.Name }}-telemetrystore-{{ $shardIdx }}-{{ $replicaIdx }}-data: + {{- end }} + {{- end }} + {{ $.Metadata.Name }}-telemetrystore-user-scripts: + {{- if ne $.Spec.MetaStore.Kind.String "sqlite" }} + {{- range $replicaIdx := until (int $.Spec.MetaStore.Spec.Cluster.Replicas) }} + {{ $.Metadata.Name }}-metastore-{{ $.Spec.MetaStore.Kind }}-{{ $replicaIdx }}-data: + {{- end }} + {{- end }} + {{- if eq $.Spec.MetaStore.Kind.String "sqlite" }} + {{- range $replicaIdx := until (int $.Spec.Signoz.Spec.Cluster.Replicas) }} + {{ $.Metadata.Name }}-metastore-{{ $.Spec.MetaStore.Kind }}-{{ $replicaIdx }}-data: + {{- end }} + {{- end }} +configs: + {{- range $replicaIdx := until (int $.Spec.TelemetryKeeper.Spec.Cluster.Replicas) }} + telemetrykeeper-{{ $replicaIdx }}: + file: telemetrykeeper/{{ $.Spec.TelemetryKeeper.Kind }}/keeper-{{ $replicaIdx }}.yaml + {{- end }} + {{- range $shardIdx := until (int $.Spec.TelemetryStore.Spec.Cluster.Shards) }} + {{- range $replicaIdx := until (int (add1 (int $.Spec.TelemetryStore.Spec.Cluster.Replicas))) }} + telemetrystore-config-{{ $shardIdx }}-{{ $replicaIdx }}: + file: telemetrystore/{{ $.Spec.TelemetryStore.Kind }}/config-{{ $shardIdx }}-{{ $replicaIdx }}.yaml + {{- end }} + {{- end }} + telemetrystore-functions: + file: telemetrystore/{{ $.Spec.TelemetryStore.Kind }}/functions.yaml + ingester-config: + file: ingester/ingester.yaml + manager-config: + file: ingester/opamp.yaml diff --git a/internal/casting/ecsterraformcasting/casting.go b/internal/casting/ecsterraformcasting/casting.go new file mode 100644 index 0000000..f47205d --- /dev/null +++ b/internal/casting/ecsterraformcasting/casting.go @@ -0,0 +1,223 @@ +package ecsterraformcasting + +import ( + "context" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/signoz/foundry/api/v1alpha1/installation" + rootcasting "github.com/signoz/foundry/internal/casting" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/molding" +) + +var _ rootcasting.Casting = (*ecsCasting)(nil) + +type ecsCasting struct { + logger *slog.Logger +} + +func New(logger *slog.Logger) *ecsCasting { + return &ecsCasting{ + logger: logger, + } +} + +func (c *ecsCasting) Enricher(ctx context.Context, config *installation.Casting) (molding.MoldingEnricher, error) { + return newEcsMoldingEnricher(config) +} + +func (c *ecsCasting) Forge(ctx context.Context, config installation.Casting, poursPath string) ([]domain.Material, error) { + var materials []domain.Material + + deployDir := rootcasting.DeploymentDir + moduleDir := filepath.Join(deployDir, "module") + + // Root Terraform files + rootTemplates := map[string]*domain.Template{ + "main.tf.json": mainTF, + "variables.tf.json": variablesTF, + "terraform.tfvars.json": tfarsTF, + } + for filename, tmpl := range rootTemplates { + m, err := tmpl.Render(config, filepath.Join(deployDir, filename)) + if err != nil { + return nil, err + } + materials = append(materials, m) + } + + // Module shared files + moduleTemplates := map[string]*domain.Template{ + "main.tf.json": moduleMainTF, + "variables.tf.json": moduleVariablesTF, + "outputs.tf.json": moduleOutputsTF, + } + for filename, tmpl := range moduleTemplates { + m, err := tmpl.Render(config, filepath.Join(moduleDir, filename)) + if err != nil { + return nil, err + } + materials = append(materials, m) + } + + // TelemetryKeeper + if config.Spec.TelemetryKeeper.Spec.IsEnabled() { + m, err := moduleTelemetryKeeperTF.Render(config, filepath.Join(moduleDir, "telemetrykeeper.tf.json")) + if err != nil { + return nil, err + } + materials = append(materials, m) + + for filename, content := range config.Spec.TelemetryKeeper.Spec.Config.Data { + material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(moduleDir, "telemetrykeeper", config.Spec.TelemetryKeeper.Kind.String(), filename)) + if err != nil { + return nil, err + } + materials = append(materials, material) + } + } + + // TelemetryStore + if config.Spec.TelemetryStore.Spec.IsEnabled() { + m, err := moduleTelemetryStoreTF.Render(config, filepath.Join(moduleDir, "telemetrystore.tf.json")) + if err != nil { + return nil, err + } + materials = append(materials, m) + + for filename, content := range config.Spec.TelemetryStore.Spec.Config.Data { + material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(moduleDir, "telemetrystore", config.Spec.TelemetryStore.Kind.String(), filename)) + if err != nil { + return nil, err + } + materials = append(materials, material) + } + } + + // TelemetryStore migrator + if config.Spec.TelemetryStore.Spec.IsEnabled() { + m, err := moduleMigratorTF.Render(config, filepath.Join(moduleDir, "telemetrystore_migrator.tf.json")) + if err != nil { + return nil, err + } + materials = append(materials, m) + } + + // MetaStore + if config.Spec.MetaStore.Spec.IsEnabled() { + m, err := moduleMetaStoreTF.Render(config, filepath.Join(moduleDir, "metastore.tf.json")) + if err != nil { + return nil, err + } + materials = append(materials, m) + + for filename, content := range config.Spec.MetaStore.Spec.Config.Data { + material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(moduleDir, "metastore", config.Spec.MetaStore.Kind.String(), filename)) + if err != nil { + return nil, err + } + materials = append(materials, material) + } + } + + // Signoz + if config.Spec.Signoz.Spec.IsEnabled() { + m, err := moduleSignozTF.Render(config, filepath.Join(moduleDir, "signoz.tf.json")) + if err != nil { + return nil, err + } + materials = append(materials, m) + } + + // Ingester + if config.Spec.Ingester.Spec.IsEnabled() { + m, err := moduleIngesterTF.Render(config, filepath.Join(moduleDir, "ingester.tf.json")) + if err != nil { + return nil, err + } + materials = append(materials, m) + + for filename, content := range config.Spec.Ingester.Spec.Config.Data { + material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(moduleDir, "ingester", filename)) + if err != nil { + return nil, err + } + materials = append(materials, material) + } + } + + return materials, nil +} + +func (c *ecsCasting) Cast(ctx context.Context, config installation.Casting, outputPath string) error { + c.logger.InfoContext(ctx, "Running Terraform for ECS deployment") + + deploymentDir := filepath.Join(outputPath, rootcasting.DeploymentDir) + + // Verify terraform files exist + if _, err := os.Stat(filepath.Join(deploymentDir, "main.tf.json")); os.IsNotExist(err) { + return errors.Newf(errors.TypeNotFound, "terraform files do not exist at path: %s; run forge first", deploymentDir) + } + + // Create a context with 10-minute timeout (terraform can be slow) + runctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + + // Run terraform init + c.logger.InfoContext(runctx, "Running terraform init") + initCmd := exec.CommandContext(runctx, "terraform", "-chdir="+deploymentDir, "init") + initCmd.Stdout = os.Stdout + initCmd.Stderr = os.Stderr + if err := initCmd.Run(); err != nil { + c.logger.ErrorContext(runctx, "terraform init failed", slog.String("error", err.Error())) + return errors.Wrapf(err, errors.TypeInternal, "terraform init failed") + } + + // Run terraform apply + c.logger.InfoContext(runctx, "Running terraform apply") + args := []string{"-chdir=" + deploymentDir, "apply", "-auto-approve"} + c.logger.DebugContext(runctx, "Running command", slog.String("command", "terraform "+strings.Join(args, " "))) + + applyCmd := exec.CommandContext(runctx, "terraform", args...) + applyCmd.Stdout = os.Stdout + applyCmd.Stderr = os.Stderr + if err := applyCmd.Run(); err != nil { + c.logger.ErrorContext(runctx, "terraform apply failed", slog.String("error", err.Error())) + return errors.Wrapf(err, errors.TypeInternal, "terraform apply failed") + } + + c.logger.InfoContext(runctx, "Terraform apply completed successfully") + return nil +} + +// getMaterials renders all module templates and returns them as JSONMaterials. +func getMaterials(config *installation.Casting) ([]domain.StructuredMaterial, error) { + var materials []domain.StructuredMaterial + + for _, tmpl := range []*domain.Template{ + moduleMainTF, + moduleTelemetryStoreTF, + moduleTelemetryKeeperTF, + moduleMetaStoreTF, + moduleSignozTF, + moduleIngesterTF, + } { + m, err := tmpl.Render(*config, tmpl.Path()) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to create material") + } + sm, ok := m.(domain.StructuredMaterial) + if !ok { + return nil, errors.Newf(errors.TypeInternal, "template %s does not produce a structured material", tmpl.Path()) + } + materials = append(materials, sm) + } + + return materials, nil +} diff --git a/internal/casting/ecsterraformcasting/embed.go b/internal/casting/ecsterraformcasting/embed.go new file mode 100644 index 0000000..3a0d80e --- /dev/null +++ b/internal/casting/ecsterraformcasting/embed.go @@ -0,0 +1,31 @@ +package ecsterraformcasting + +import ( + "embed" + + "github.com/signoz/foundry/internal/domain" +) + +//go:embed templates/*.gotmpl templates/module/*.gotmpl +var templates embed.FS + +// Root Terraform templates. +var ( + mainTF = domain.MustNewTemplateFromFS(templates, "templates/main.tf.json.gotmpl", domain.FormatJSON) + variablesTF = domain.MustNewTemplateFromFS(templates, "templates/variables.tf.json.gotmpl", domain.FormatJSON) + tfarsTF = domain.MustNewTemplateFromFS(templates, "templates/terraform.tfvars.json.gotmpl", domain.FormatJSON) +) + +// Module Terraform templates. +var ( + moduleMainTF = domain.MustNewTemplateFromFS(templates, "templates/module/main.tf.json.gotmpl", domain.FormatJSON) + moduleVariablesTF = domain.MustNewTemplateFromFS(templates, "templates/module/variables.tf.json.gotmpl", domain.FormatJSON) + moduleOutputsTF = domain.MustNewTemplateFromFS(templates, "templates/module/outputs.tf.json.gotmpl", domain.FormatJSON) + + moduleTelemetryKeeperTF = domain.MustNewTemplateFromFS(templates, "templates/module/telemetrykeeper.tf.json.gotmpl", domain.FormatJSON) + moduleTelemetryStoreTF = domain.MustNewTemplateFromFS(templates, "templates/module/telemetrystore.tf.json.gotmpl", domain.FormatJSON) + moduleMigratorTF = domain.MustNewTemplateFromFS(templates, "templates/module/telemetrystore_migrator.tf.json.gotmpl", domain.FormatJSON) + moduleMetaStoreTF = domain.MustNewTemplateFromFS(templates, "templates/module/metastore.tf.json.gotmpl", domain.FormatJSON) + moduleSignozTF = domain.MustNewTemplateFromFS(templates, "templates/module/signoz.tf.json.gotmpl", domain.FormatJSON) + moduleIngesterTF = domain.MustNewTemplateFromFS(templates, "templates/module/ingester.tf.json.gotmpl", domain.FormatJSON) +) diff --git a/internal/casting/ecsterraformcasting/embed_test.go b/internal/casting/ecsterraformcasting/embed_test.go new file mode 100644 index 0000000..060c348 --- /dev/null +++ b/internal/casting/ecsterraformcasting/embed_test.go @@ -0,0 +1,66 @@ +package ecsterraformcasting + +import ( + "bytes" + "testing" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/installation" + "github.com/signoz/foundry/internal/domain" + "github.com/stretchr/testify/assert" +) + +func TestNotEmptyAndValid(t *testing.T) { + templates := map[string]*domain.Template{ + "mainTF": mainTF, + "variablesTF": variablesTF, + "moduleMainTF": moduleMainTF, + "moduleVariablesTF": moduleVariablesTF, + "moduleOutputsTF": moduleOutputsTF, + "moduleTelemetryKeeperTF": moduleTelemetryKeeperTF, + "moduleTelemetryStoreTF": moduleTelemetryStoreTF, + "moduleMigratorTF": moduleMigratorTF, + "moduleMetaStoreTF": moduleMetaStoreTF, + "moduleSignozTF": moduleSignozTF, + "moduleIngesterTF": moduleIngesterTF, + } + + for name, tmpl := range templates { + assert.NotEmpty(t, tmpl, "%s should not be empty", name) + buf := bytes.NewBuffer(nil) + err := tmpl.Execute(buf, nil) + assert.NoError(t, err, "error executing %s", name) + assert.NotEmpty(t, buf.String(), "%s output should not be empty", name) + } +} + +func TestTfvarsTemplateWithAnnotations(t *testing.T) { + assert.NotEmpty(t, tfarsTF) + + casting := &installation.Casting{ + CastingMeta: v1alpha1.CastingMeta{ + Metadata: v1alpha1.TypeMetadata{ + Name: "signoz", + Annotations: map[string]string{ + "foundry.signoz.io/ecs/region": "us-east-1", + "foundry.signoz.io/ecs/cluster-id": "arn:aws:ecs:us-east-1:123456789012:cluster/test", + "foundry.signoz.io/ecs/subnet-ids": "subnet-abc123,subnet-def456", + "foundry.signoz.io/ecs/security-group-ids": "sg-abc123", + "foundry.signoz.io/ecs/vpc-id": "vpc-abc123", + "foundry.signoz.io/ecs/config-bucket": "test-configs", + "foundry.signoz.io/ecs/task-role-arn": "arn:aws:iam::123456789012:role/task", + "foundry.signoz.io/ecs/task-execution-role-arn": "arn:aws:iam::123456789012:role/exec", + "foundry.signoz.io/ecs/capacity-provider": "test-provider", + }, + }, + }, + } + + buf := bytes.NewBuffer(nil) + err := tfarsTF.Execute(buf, casting) + assert.NoError(t, err) + assert.NotEmpty(t, buf.String()) + assert.Contains(t, buf.String(), "us-east-1") + assert.Contains(t, buf.String(), "subnet-abc123") + assert.Contains(t, buf.String(), "subnet-def456") +} diff --git a/internal/casting/ecsterraformcasting/enricher.go b/internal/casting/ecsterraformcasting/enricher.go new file mode 100644 index 0000000..6086149 --- /dev/null +++ b/internal/casting/ecsterraformcasting/enricher.go @@ -0,0 +1,82 @@ +package ecsterraformcasting + +import ( + "context" + "fmt" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/installation" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/molding" +) + +const ( + telemetryStorePort = 9000 + telemetryKeeperClientPort = 9181 + telemetryKeeperRaftPort = 9234 + metaStorePort = 5432 + signozAPIPort = 8080 + signozOpampPort = 4320 +) + +var _ molding.MoldingEnricher = (*ecsMoldingEnricher)(nil) + +type ecsMoldingEnricher struct { + materials []domain.StructuredMaterial +} + +func newEcsMoldingEnricher(config *installation.Casting) (*ecsMoldingEnricher, error) { + materials, err := getMaterials(config) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to get materials") + } + + return &ecsMoldingEnricher{materials: materials}, nil +} + +func (enricher *ecsMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *installation.Casting) error { + namespaceBytes, err := enricher.materials[0].GetBytes("resource.aws_service_discovery_private_dns_namespace.main.name") + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to get namespace") + } + namespace := string(namespaceBytes) + + switch kind { + case v1alpha1.MoldingKindTelemetryStore: + sdName, err := enricher.materials[1].GetBytes("resource.aws_service_discovery_service.telemetrystore.name") + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to get telemetrystore service discovery name") + } + fqdn := fmt.Sprintf("%s.%s", string(sdName), namespace) + config.Spec.TelemetryStore.Status.Addresses.TCP = []string{domain.MustNewAddress("tcp", fqdn, telemetryStorePort).String()} + + case v1alpha1.MoldingKindTelemetryKeeper: + sdName, err := enricher.materials[2].GetBytes("resource.aws_service_discovery_service.telemetrykeeper.name") + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to get telemetrykeeper service discovery name") + } + fqdn := fmt.Sprintf("%s.%s", string(sdName), namespace) + config.Spec.TelemetryKeeper.Status.Addresses.Client = []string{domain.MustNewAddress("tcp", fqdn, telemetryKeeperClientPort).String()} + config.Spec.TelemetryKeeper.Status.Addresses.Raft = []string{domain.MustNewAddress("tcp", fqdn, telemetryKeeperRaftPort).String()} + + case v1alpha1.MoldingKindMetaStore: + sdName, err := enricher.materials[3].GetBytes("resource.aws_service_discovery_service.metastore.name") + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to get metastore service discovery name") + } + fqdn := fmt.Sprintf("%s.%s", string(sdName), namespace) + config.Spec.MetaStore.Status.Addresses.DSN = []string{domain.MustNewAddress("tcp", fqdn, metaStorePort).String()} + + case v1alpha1.MoldingKindSignoz: + sdName, err := enricher.materials[4].GetBytes("resource.aws_service_discovery_service.signoz.name") + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to get signoz service discovery name") + } + fqdn := fmt.Sprintf("%s.%s", string(sdName), namespace) + config.Spec.Signoz.Status.Addresses.APIServer = []string{domain.MustNewAddress("tcp", fqdn, signozAPIPort).String()} + config.Spec.Signoz.Status.Addresses.Opamp = []string{domain.MustNewAddress("ws", fqdn, signozOpampPort).String()} + } + + return nil +} diff --git a/internal/casting/ecsterraformcasting/templates/main.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/main.tf.json.gotmpl new file mode 100644 index 0000000..7cdd6a4 --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/main.tf.json.gotmpl @@ -0,0 +1,30 @@ +{ + "terraform": { + "required_version": ">= 1.0", + "required_providers": { + "aws": { + "source": "hashicorp/aws", + "version": ">= 5.0" + } + } + }, + "provider": { + "aws": { + "region": "${var.region}" + } + }, + "module": { + "signoz": { + "source": "./module", + "region": "${var.region}", + "ecs_cluster_id": "${var.ecs_cluster_id}", + "subnet_ids": "${var.subnet_ids}", + "security_group_ids": "${var.security_group_ids}", + "vpc_id": "${var.vpc_id}", + "config_bucket": "${var.config_bucket}", + "task_role_arn": "${var.task_role_arn}", + "task_execution_role_arn": "${var.task_execution_role_arn}", + "capacity_provider": "${var.capacity_provider}" + } + } +} diff --git a/internal/casting/ecsterraformcasting/templates/module/ingester.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/module/ingester.tf.json.gotmpl new file mode 100644 index 0000000..cb3946c --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/module/ingester.tf.json.gotmpl @@ -0,0 +1,133 @@ +{{- $name := $.Metadata.Name -}} +{ + "locals": { + "containers": [ + { + "name": "config-fetcher", + "image": "amazon/aws-cli:2.27.32", + "essential": false, + "entryPoint": ["/bin/sh", "-c"], + "command": ["aws s3 cp s3://${var.config_bucket}/{{ $name }}/ingester/ /configs/ingester/ --recursive"], + "mountPoints": [ + { + "sourceVolume": "ingester-config", + "containerPath": "/configs/ingester" + } + ], + "cpu": 10, + "memory": 102, + "memoryReservation": 102 + }, + { + "name": "ingester", + "image": "{{ $.Spec.Ingester.Spec.Image }}", + "essential": true, + "entryPoint": ["/bin/sh", "-c"], + "command": ["/signoz-otel-collector migrate sync check && /signoz-otel-collector --config=/conf/ingester.yaml --manager-config=/conf/opamp.yaml --copy-path=/var/tmp/collector-config.yaml"], + {{- $env := list }} + {{- if and $.Spec.TelemetryStore.Spec.Enabled $.Spec.TelemetryStore.Status.Addresses.TCP (index $.Spec.TelemetryStore.Status.Addresses.TCP 0) }} + {{- $env = append $env (dict "name" "SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN" "value" (index $.Spec.TelemetryStore.Status.Addresses.TCP 0)) }} + {{- end }} + {{- range $key, $value := $.Spec.Ingester.Spec.Env }} + {{- $env = append $env (dict "name" $key "value" $value) }} + {{- end }} + "environment": {{ toJson $env }}, + "portMappings": [ + {"name": "grpc", "containerPort": 4317, "protocol": "tcp", "appProtocol": "grpc"}, + {"name": "http", "containerPort": 4318, "protocol": "tcp", "appProtocol": "http"} + ], + "mountPoints": [ + { + "sourceVolume": "ingester-config", + "containerPath": "/conf" + } + ], + "dependsOn": [ + {"containerName": "config-fetcher", "condition": "SUCCESS"} + ], + "cpu": 512, + "memory": 512, + "memoryReservation": 512, + "healthCheck": { + "command": ["CMD-SHELL", "wget --spider -q localhost:13133 || exit 1"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + } + } + ] + }, + "resource": { + "aws_s3_object": { + "ingester_configs": { + "for_each": "${fileset(\"${path.module}/ingester\", \"*.yaml\")}", + "bucket": "${var.config_bucket}", + "key": "{{ $name }}/ingester/${each.value}", + "source": "${path.module}/ingester/${each.value}", + "etag": "${filemd5(\"${path.module}/ingester/${each.value}\")}" + } + }, + "aws_ecs_task_definition": { + "ingester": { + "family": "{{ $name }}-ingester", + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${var.task_role_arn}", + "execution_role_arn": "${var.task_execution_role_arn}", + "container_definitions": "${jsonencode(local.containers)}", + "volume": [ + { + "name": "ingester-config", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + } + ], + "depends_on": ["aws_s3_object.ingester_configs"] + } + }, + "aws_service_discovery_service": { + "ingester": { + "name": "ingester", + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + }, + "aws_ecs_service": { + "ingester": { + "name": "{{ $name }}-ingester", + "cluster": "${var.ecs_cluster_id}", + "task_definition": "${aws_ecs_task_definition.ingester.arn}", + "desired_count": 1, + "capacity_provider_strategy": [ + { + "capacity_provider": "${var.capacity_provider}", + "weight": 1, + "base": 0 + } + ], + "network_configuration": { + "subnets": "${var.subnet_ids}", + "security_groups": "${var.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.ingester.arn}" + }, +"depends_on": [ + "aws_ecs_service.signoz", + "aws_ecs_service.telemetrystore" + ] + } + } + } +} diff --git a/internal/casting/ecsterraformcasting/templates/module/main.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/module/main.tf.json.gotmpl new file mode 100644 index 0000000..db84360 --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/module/main.tf.json.gotmpl @@ -0,0 +1,11 @@ +{{- $name := $.Metadata.Name -}} +{ + "resource": { + "aws_service_discovery_private_dns_namespace": { + "main": { + "name": "{{ $name }}.local", + "vpc": "${var.vpc_id}" + } + } + } +} diff --git a/internal/casting/ecsterraformcasting/templates/module/metastore.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/module/metastore.tf.json.gotmpl new file mode 100644 index 0000000..2400231 --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/module/metastore.tf.json.gotmpl @@ -0,0 +1,94 @@ +{{- $name := $.Metadata.Name -}} +{ + "locals": { + "containers": [ + { + "name": "{{ $name }}-metastore-{{ $.Spec.MetaStore.Kind }}-0", + "image": "{{ $.Spec.MetaStore.Spec.Image }}", + "essential": true, + {{- $env := list }} + {{- range $key, $value := $.Spec.MetaStore.Spec.Env }} + {{- $env = append $env (dict "name" $key "value" $value) }} + {{- end }} + "environment": {{ toJson $env }}, + "portMappings": [ + {"name": "postgres", "containerPort": 5432, "protocol": "tcp"} + ], + "mountPoints": [ + { + "sourceVolume": "postgres-data", + "containerPath": "/var/lib/postgresql/data" + } + ], + "cpu": 256, + "memory": 256, + "memoryReservation": 256, + "healthCheck": { + "command": ["CMD-SHELL", "pg_isready -U postgres || exit 1"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + } + } + ] + }, + "resource": { + "aws_ecs_task_definition": { + "metastore": { + "family": "{{ $name }}-metastore-{{ $.Spec.MetaStore.Kind }}-0", + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${var.task_role_arn}", + "execution_role_arn": "${var.task_execution_role_arn}", + "container_definitions": "${jsonencode(local.containers)}", + "volume": [ + { + "name": "postgres-data", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + } + ] + } + }, + "aws_service_discovery_service": { + "metastore": { + "name": "metastore-{{ $.Spec.MetaStore.Kind }}", + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + }, + "aws_ecs_service": { + "metastore": { + "name": "{{ $name }}-metastore-{{ $.Spec.MetaStore.Kind }}-0", + "cluster": "${var.ecs_cluster_id}", + "task_definition": "${aws_ecs_task_definition.metastore.arn}", + "desired_count": 1, + "capacity_provider_strategy": [ + { + "capacity_provider": "${var.capacity_provider}", + "weight": 1, + "base": 0 + } + ], + "network_configuration": { + "subnets": "${var.subnet_ids}", + "security_groups": "${var.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.metastore.arn}" + } + } + } + } +} diff --git a/internal/casting/ecsterraformcasting/templates/module/outputs.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/module/outputs.tf.json.gotmpl new file mode 100644 index 0000000..4df0dcb --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/module/outputs.tf.json.gotmpl @@ -0,0 +1,62 @@ +{ + "output": { + "ecs_cluster_id": { + "description": "ECS cluster ID", + "value": "${var.ecs_cluster_id}" + }, + "namespace_id": { + "description": "Cloud Map private DNS namespace ID", + "value": "${aws_service_discovery_private_dns_namespace.main.id}" + }, + "namespace_name": { + "description": "Cloud Map private DNS namespace name", + "value": "${aws_service_discovery_private_dns_namespace.main.name}" + }, + "subnet_ids": { + "description": "Subnet IDs used by ECS services", + "value": "${var.subnet_ids}" + }, + "security_group_ids": { + "description": "Security group IDs used by ECS services", + "value": "${var.security_group_ids}" + } + {{- if $.Spec.Signoz.Spec.Enabled }}, + "signoz_service_arn": { + "description": "SigNoz ECS service ARN (target for ALB on port 8080)", + "value": "${aws_ecs_service.signoz.id}" + }, + "signoz_service_name": { + "description": "SigNoz ECS service name", + "value": "${aws_ecs_service.signoz.name}" + } + {{- end }} + {{- if $.Spec.Ingester.Spec.Enabled }}, + "ingester_service_arn": { + "description": "Ingester ECS service ARN (target for NLB on port 4317/4318)", + "value": "${aws_ecs_service.ingester.id}" + }, + "ingester_service_name": { + "description": "Ingester ECS service name", + "value": "${aws_ecs_service.ingester.name}" + } + {{- end }} + {{- if $.Spec.TelemetryStore.Spec.Enabled }}, + "telemetrystore_service_name": { + "description": "TelemetryStore ECS service name", + "value": "${aws_ecs_service.telemetrystore.name}" + } + {{- end }} + {{- if $.Spec.TelemetryKeeper.Spec.Enabled }}, + "telemetrykeeper_service_name": { + "description": "TelemetryKeeper ECS service name", + "value": "${aws_ecs_service.telemetrykeeper.name}" + } + {{- end }} + {{- if $.Spec.MetaStore.Spec.Enabled }}, + "metastore_service_name": { + "description": "MetaStore ECS service name", + "value": "${aws_ecs_service.metastore.name}" + } + {{- end }} + } +} diff --git a/internal/casting/ecsterraformcasting/templates/module/signoz.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/module/signoz.tf.json.gotmpl new file mode 100644 index 0000000..07822a0 --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/module/signoz.tf.json.gotmpl @@ -0,0 +1,86 @@ +{{- $name := $.Metadata.Name -}} +{ + "locals": { + "containers": [ + { + "name": "signoz", + "image": "{{ $.Spec.Signoz.Spec.Image }}", + "essential": true, + {{- $env := list }} + {{- range $key, $value := $.Spec.Signoz.Spec.Env }} + {{- $env = append $env (dict "name" $key "value" $value) }} + {{- end }} + {{- if $env }} + "environment": {{ toJson $env }}, + {{- end }} + "portMappings": [ + {"name": "http", "containerPort": 8080, "protocol": "tcp", "appProtocol": "http"}, + {"name": "opamp", "containerPort": 4320, "protocol": "tcp"} + ], + "cpu": 512, + "memory": 512, + "memoryReservation": 512, + "healthCheck": { + "command": ["CMD-SHELL", "wget --spider -q localhost:8080/api/v1/health || exit 1"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + } + } + ] + }, + "resource": { + "aws_ecs_task_definition": { + "signoz": { + "family": "{{ $name }}-signoz", + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${var.task_role_arn}", + "execution_role_arn": "${var.task_execution_role_arn}", + "container_definitions": "${jsonencode(local.containers)}" + } + }, + "aws_service_discovery_service": { + "signoz": { + "name": "signoz", + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + }, + "aws_ecs_service": { + "signoz": { + "name": "{{ $name }}-signoz", + "cluster": "${var.ecs_cluster_id}", + "task_definition": "${aws_ecs_task_definition.signoz.arn}", + "desired_count": 1, + "capacity_provider_strategy": [ + { + "capacity_provider": "${var.capacity_provider}", + "weight": 1, + "base": 0 + } + ], + "network_configuration": { + "subnets": "${var.subnet_ids}", + "security_groups": "${var.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.signoz.arn}" + }, +"depends_on": [ + "aws_ecs_service.metastore", + "aws_ecs_service.telemetrystore" + ] + } + } + } +} diff --git a/internal/casting/ecsterraformcasting/templates/module/telemetrykeeper.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/module/telemetrykeeper.tf.json.gotmpl new file mode 100644 index 0000000..efaffc4 --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/module/telemetrykeeper.tf.json.gotmpl @@ -0,0 +1,138 @@ +{{- $name := $.Metadata.Name -}} +{ + "locals": { + "containers": [ + { + "name": "config-fetcher", + "image": "amazon/aws-cli:2.27.32", + "essential": false, + "entryPoint": ["/bin/sh", "-c"], + "command": ["aws s3 cp s3://${var.config_bucket}/{{ $name }}/telemetrykeeper/{{ $.Spec.TelemetryKeeper.Kind }}/ /configs/telemetrykeeper/ --recursive"], + "mountPoints": [ + { + "sourceVolume": "keeper-config", + "containerPath": "/configs/telemetrykeeper" + } + ], + "cpu": 10, + "memory": 102, + "memoryReservation": 102 + }, + { + "name": "{{ $name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-0", + "image": "{{ $.Spec.TelemetryKeeper.Spec.Image }}", + "essential": true, + "entryPoint": ["/usr/bin/clickhouse-keeper", "--config-file=/etc/clickhouse-keeper/keeper.yaml"], + {{- $env := list }} + {{- range $key, $value := $.Spec.TelemetryKeeper.Spec.Env }} + {{- $env = append $env (dict "name" $key "value" $value) }} + {{- end }} + {{- if $env }} + "environment": {{ toJson $env }}, + {{- end }} + "portMappings": [ + {"name": "client", "containerPort": 9181, "protocol": "tcp"}, + {"name": "raft", "containerPort": 9234, "protocol": "tcp"} + ], + "mountPoints": [ + { + "sourceVolume": "keeper-data", + "containerPath": "/var/lib/clickhouse-keeper" + }, + { + "sourceVolume": "keeper-config", + "containerPath": "/etc/clickhouse-keeper" + } + ], + "dependsOn": [ + {"containerName": "config-fetcher", "condition": "SUCCESS"} + ], + "cpu": 256, + "memory": 512, + "memoryReservation": 512, + "healthCheck": { + "command": ["CMD-SHELL", "echo ruok | nc localhost 9181 || exit 1"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + } + } + ] + }, + "resource": { + "aws_s3_object": { + "telemetrykeeper_configs": { + "for_each": "${fileset(\"${path.module}/telemetrykeeper/{{ $.Spec.TelemetryKeeper.Kind }}\", \"**\")}", + "bucket": "${var.config_bucket}", + "key": "{{ $name }}/telemetrykeeper/{{ $.Spec.TelemetryKeeper.Kind }}/${each.value}", + "source": "${path.module}/telemetrykeeper/{{ $.Spec.TelemetryKeeper.Kind }}/${each.value}", + "etag": "${filemd5(\"${path.module}/telemetrykeeper/{{ $.Spec.TelemetryKeeper.Kind }}/${each.value}\")}" + } + }, + "aws_ecs_task_definition": { + "telemetrykeeper": { + "family": "{{ $name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-0", + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${var.task_role_arn}", + "execution_role_arn": "${var.task_execution_role_arn}", + "container_definitions": "${jsonencode(local.containers)}", + "volume": [ + { + "name": "keeper-config", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + }, + { + "name": "keeper-data", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + } + ], + "depends_on": ["aws_s3_object.telemetrykeeper_configs"] + } + }, + "aws_service_discovery_service": { + "telemetrykeeper": { + "name": "telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}", + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + }, + "aws_ecs_service": { + "telemetrykeeper": { + "name": "{{ $name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-0", + "cluster": "${var.ecs_cluster_id}", + "task_definition": "${aws_ecs_task_definition.telemetrykeeper.arn}", + "desired_count": 1, + "capacity_provider_strategy": [ + { + "capacity_provider": "${var.capacity_provider}", + "weight": 1, + "base": 0 + } + ], + "network_configuration": { + "subnets": "${var.subnet_ids}", + "security_groups": "${var.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.telemetrykeeper.arn}" + } + } + } + } +} diff --git a/internal/casting/ecsterraformcasting/templates/module/telemetrystore.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/module/telemetrystore.tf.json.gotmpl new file mode 100644 index 0000000..372e3e1 --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/module/telemetrystore.tf.json.gotmpl @@ -0,0 +1,166 @@ +{{- $name := $.Metadata.Name -}} +{ + "locals": { + "containers": [ + { + "name": "init-clickhouse", + "image": "alpine:3.18.2", + "essential": false, + "command": ["/bin/sh", "-c", "cd /tmp && wget -O histogram-quantile.tar.gz 'https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2Fv0.0.1/histogram-quantile_linux_amd64.tar.gz' && tar -xzf histogram-quantile.tar.gz && mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile"], + "mountPoints": [ + { + "sourceVolume": "shared-binary-volume", + "containerPath": "/var/lib/clickhouse/user_scripts" + } + ], + "cpu": 256, + "memory": 256, + "memoryReservation": 256 + }, + { + "name": "config-fetcher", + "image": "amazon/aws-cli:2.27.32", + "essential": false, + "entryPoint": ["/bin/sh", "-c"], + "command": ["aws s3 cp s3://${var.config_bucket}/{{ $name }}/telemetrystore/{{ $.Spec.TelemetryStore.Kind }}/ /configs/telemetrystore/ --recursive"], + "mountPoints": [ + { + "sourceVolume": "telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-config", + "containerPath": "/configs/telemetrystore" + } + ], + "cpu": 10, + "memory": 102, + "memoryReservation": 102 + }, + { + "name": "{{ $name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-0-0", + "image": "{{ $.Spec.TelemetryStore.Spec.Image }}", + "essential": true, + {{- $env := list (dict "name" "CLICKHOUSE_SKIP_USER_SETUP" "value" "1") }} + {{- range $key, $value := $.Spec.TelemetryStore.Spec.Env }} + {{- $env = append $env (dict "name" $key "value" $value) }} + {{- end }} + "environment": {{ toJson $env }}, + "portMappings": [ + {"name": "native", "containerPort": 9000, "protocol": "tcp"}, + {"name": "http", "containerPort": 8123, "protocol": "tcp", "appProtocol": "http"}, + {"name": "prometheus", "containerPort": 9363, "protocol": "tcp", "appProtocol": "http"} + ], + "mountPoints": [ + { + "sourceVolume": "shared-binary-volume", + "containerPath": "/var/lib/clickhouse/user_scripts" + }, + { + "sourceVolume": "telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-config", + "containerPath": "/etc/clickhouse-server/config.d" + }, + { + "sourceVolume": "clickhouse-data", + "containerPath": "/var/lib/clickhouse" + } + ], + "dependsOn": [ + {"containerName": "init-clickhouse", "condition": "SUCCESS"}, + {"containerName": "config-fetcher", "condition": "SUCCESS"} + ], + "cpu": 1024, + "memory": 512, + "memoryReservation": 512, + "healthCheck": { + "command": ["CMD-SHELL", "wget --spider -q 0.0.0.0:8123/ping || exit 1"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + } + } + ] + }, + "resource": { + "aws_s3_object": { + "telemetrystore_configs": { + "for_each": "${fileset(\"${path.module}/telemetrystore/{{ $.Spec.TelemetryStore.Kind }}\", \"**\")}", + "bucket": "${var.config_bucket}", + "key": "{{ $name }}/telemetrystore/{{ $.Spec.TelemetryStore.Kind }}/${each.value}", + "source": "${path.module}/telemetrystore/{{ $.Spec.TelemetryStore.Kind }}/${each.value}", + "etag": "${filemd5(\"${path.module}/telemetrystore/{{ $.Spec.TelemetryStore.Kind }}/${each.value}\")}" + } + }, + "aws_ecs_task_definition": { + "telemetrystore": { + "family": "{{ $name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-0-0", + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${var.task_role_arn}", + "execution_role_arn": "${var.task_execution_role_arn}", + "container_definitions": "${jsonencode(local.containers)}", + "volume": [ + { + "name": "shared-binary-volume", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + }, + { + "name": "telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-config", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + }, + { + "name": "clickhouse-data", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + } + ], + "depends_on": ["aws_s3_object.telemetrystore_configs"] + } + }, + "aws_service_discovery_service": { + "telemetrystore": { + "name": "telemetrystore-{{ $.Spec.TelemetryStore.Kind }}", + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + }, + "aws_ecs_service": { + "telemetrystore": { + "name": "{{ $name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-0-0", + "cluster": "${var.ecs_cluster_id}", + "task_definition": "${aws_ecs_task_definition.telemetrystore.arn}", + "desired_count": 1, + "capacity_provider_strategy": [ + { + "capacity_provider": "${var.capacity_provider}", + "weight": 1, + "base": 0 + } + ], + "network_configuration": { + "subnets": "${var.subnet_ids}", + "security_groups": "${var.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.telemetrystore.arn}" + }, +"depends_on": [ + "aws_ecs_service.telemetrykeeper" + ] + } + } + } +} diff --git a/internal/casting/ecsterraformcasting/templates/module/telemetrystore_migrator.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/module/telemetrystore_migrator.tf.json.gotmpl new file mode 100644 index 0000000..c1a1bd7 --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/module/telemetrystore_migrator.tf.json.gotmpl @@ -0,0 +1,56 @@ +{{- $name := $.Metadata.Name -}} +{ + "locals": { + "containers": [ + { + "name": "telemetrystore-migrator", + "image": "{{ $.Spec.Ingester.Spec.Image }}", + "essential": true, + "entryPoint": ["/bin/sh", "-c"], + "command": ["/signoz-otel-collector migrate ready && /signoz-otel-collector migrate bootstrap && /signoz-otel-collector migrate sync up && /signoz-otel-collector migrate async up"], + {{- $env := list }} + {{- if and $.Spec.TelemetryStore.Spec.Enabled $.Spec.TelemetryStore.Status.Addresses.TCP (index $.Spec.TelemetryStore.Status.Addresses.TCP 0) }} + {{- $env = append $env (dict "name" "SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN" "value" (index $.Spec.TelemetryStore.Status.Addresses.TCP 0)) }} + {{- end }} + {{- range $key, $value := $.Spec.Ingester.Spec.Env }} + {{- $env = append $env (dict "name" $key "value" $value) }} + {{- end }} + "environment": {{ toJson $env }}, + "cpu": 256, + "memory": 512 + } + ] + }, + "resource": { + "aws_ecs_task_definition": { + "telemetrystore_migrator": { + "family": "{{ $name }}-telemetrystore-migrator", + "network_mode": "awsvpc", + "requires_compatibilities": ["FARGATE"], + "cpu": 256, + "memory": 512, + "task_role_arn": "${var.task_role_arn}", + "execution_role_arn": "${var.task_execution_role_arn}", + "container_definitions": "${jsonencode(local.containers)}" + } + } + }, + "data": { + "aws_ecs_task_execution": { + "telemetrystore_migrator": { + "cluster": "${var.ecs_cluster_id}", + "task_definition": "${aws_ecs_task_definition.telemetrystore_migrator.arn}", + "desired_count": 1, + "launch_type": "FARGATE", + "network_configuration": { + "subnets": "${var.subnet_ids}", + "security_groups": "${var.security_group_ids}", + "assign_public_ip": false + }, + "depends_on": [ + "aws_ecs_service.telemetrystore" + ] + } + } + } +} diff --git a/internal/casting/ecsterraformcasting/templates/module/variables.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/module/variables.tf.json.gotmpl new file mode 100644 index 0000000..fd7e081 --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/module/variables.tf.json.gotmpl @@ -0,0 +1,40 @@ +{ + "variable": { + "region": { + "description": "AWS region", + "type": "string" + }, + "ecs_cluster_id": { + "description": "ID of the existing ECS cluster to deploy services into", + "type": "string" + }, + "subnet_ids": { + "description": "List of subnet IDs for ECS service networking (awsvpc)", + "type": "list(string)" + }, + "security_group_ids": { + "description": "List of security group IDs for ECS service networking (awsvpc)", + "type": "list(string)" + }, + "vpc_id": { + "description": "VPC ID for the private DNS namespace", + "type": "string" + }, + "config_bucket": { + "description": "S3 bucket name for storing config files", + "type": "string" + }, + "task_role_arn": { + "description": "IAM role ARN for ECS tasks", + "type": "string" + }, + "task_execution_role_arn": { + "description": "IAM role ARN for ECS task execution (pull images, write logs)", + "type": "string" + }, + "capacity_provider": { + "description": "Name of the ECS capacity provider", + "type": "string" + } + } +} diff --git a/internal/casting/ecsterraformcasting/templates/terraform.tfvars.json.gotmpl b/internal/casting/ecsterraformcasting/templates/terraform.tfvars.json.gotmpl new file mode 100644 index 0000000..867159c --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/terraform.tfvars.json.gotmpl @@ -0,0 +1,20 @@ +{{- $region := index $.Metadata.Annotations "foundry.signoz.io/ecs/region" -}} +{{- $configBucket := index $.Metadata.Annotations "foundry.signoz.io/ecs/config-bucket" -}} +{{- $taskRoleArn := index $.Metadata.Annotations "foundry.signoz.io/ecs/task-role-arn" -}} +{{- $taskExecutionRoleArn := index $.Metadata.Annotations "foundry.signoz.io/ecs/task-execution-role-arn" -}} +{{- $clusterID := index $.Metadata.Annotations "foundry.signoz.io/ecs/cluster-id" -}} +{{- $subnetIDs := index $.Metadata.Annotations "foundry.signoz.io/ecs/subnet-ids" -}} +{{- $securityGroupIDs := index $.Metadata.Annotations "foundry.signoz.io/ecs/security-group-ids" -}} +{{- $vpcID := index $.Metadata.Annotations "foundry.signoz.io/ecs/vpc-id" -}} +{{- $capacityProvider := index $.Metadata.Annotations "foundry.signoz.io/ecs/capacity-provider" -}} +{ + "region": "{{ $region }}", + "ecs_cluster_id": "{{ $clusterID }}", + "subnet_ids": [{{ range $i, $v := splitList "," $subnetIDs }}{{ if $i }}, {{ end }}"{{ trim $v }}"{{ end }}], + "security_group_ids": [{{ range $i, $v := splitList "," $securityGroupIDs }}{{ if $i }}, {{ end }}"{{ trim $v }}"{{ end }}], + "vpc_id": "{{ $vpcID }}", + "config_bucket": "{{ $configBucket }}", + "task_role_arn": "{{ $taskRoleArn }}", + "task_execution_role_arn": "{{ $taskExecutionRoleArn }}", + "capacity_provider": "{{ $capacityProvider }}" +} diff --git a/internal/casting/ecsterraformcasting/templates/variables.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/variables.tf.json.gotmpl new file mode 100644 index 0000000..fd7e081 --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/variables.tf.json.gotmpl @@ -0,0 +1,40 @@ +{ + "variable": { + "region": { + "description": "AWS region", + "type": "string" + }, + "ecs_cluster_id": { + "description": "ID of the existing ECS cluster to deploy services into", + "type": "string" + }, + "subnet_ids": { + "description": "List of subnet IDs for ECS service networking (awsvpc)", + "type": "list(string)" + }, + "security_group_ids": { + "description": "List of security group IDs for ECS service networking (awsvpc)", + "type": "list(string)" + }, + "vpc_id": { + "description": "VPC ID for the private DNS namespace", + "type": "string" + }, + "config_bucket": { + "description": "S3 bucket name for storing config files", + "type": "string" + }, + "task_role_arn": { + "description": "IAM role ARN for ECS tasks", + "type": "string" + }, + "task_execution_role_arn": { + "description": "IAM role ARN for ECS task execution (pull images, write logs)", + "type": "string" + }, + "capacity_provider": { + "description": "Name of the ECS capacity provider", + "type": "string" + } + } +} diff --git a/internal/casting/installation/planner.go b/internal/casting/installation/planner.go new file mode 100644 index 0000000..f56c594 --- /dev/null +++ b/internal/casting/installation/planner.go @@ -0,0 +1,115 @@ +package installation + +import ( + "context" + "log/slog" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/installation" + "github.com/signoz/foundry/internal/casting" + "github.com/signoz/foundry/internal/domain" + foundryerrors "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/molding" + "github.com/signoz/foundry/internal/molding/ingestermolding" + "github.com/signoz/foundry/internal/molding/metastoremolding" + "github.com/signoz/foundry/internal/molding/signozmolding" + "github.com/signoz/foundry/internal/molding/telemetrykeepermolding" + "github.com/signoz/foundry/internal/molding/telemetrystoremolding" + "github.com/signoz/foundry/internal/planner" + "github.com/signoz/foundry/internal/tooler" + "github.com/signoz/foundry/internal/tooler/terraformtooler" +) + +var _ planner.Planner = (*Planner)(nil) + +// Planner is the Installation Kind's per-Kind orchestrator. It satisfies the +// foundry planner contract by exposing this Kind's moldings, enricher, and +// casting strategy as verbs on a single value. +type Planner struct { + config *installation.Casting + logger *slog.Logger + casting casting.Casting + toolers []tooler.Tooler + enricher molding.MoldingEnricher + moldings []molding.Molding +} + +func NewPlanner(ctx context.Context, c *installation.Casting, logger *slog.Logger) (planner.Planner, error) { + registry := NewRegistry(logger) + + castingStrategy, err := registry.Casting(c.Spec.Deployment) + if err != nil { + return nil, err + } + + toolers, err := registry.Toolers(c.Spec.Deployment) + if err != nil { + return nil, err + } + + enricher, err := castingStrategy.Enricher(ctx, c) + if err != nil { + return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to get molding enricher") + } + + moldings := []molding.Molding{ + telemetrykeepermolding.New(logger), + telemetrystoremolding.New(logger), + metastoremolding.New(logger), + signozmolding.New(logger), + ingestermolding.New(logger), + } + + return &Planner{ + config: c, + logger: logger, + casting: castingStrategy, + toolers: toolers, + enricher: enricher, + moldings: moldings, + }, nil +} + +func (p *Planner) Machinery() v1alpha1.Machinery { return p.config } +func (p *Planner) Patches() []v1alpha1.PatchEntry { return p.config.Spec.Patches } + +func (p *Planner) MoldingKinds() []v1alpha1.MoldingKind { + kinds := make([]v1alpha1.MoldingKind, len(p.moldings)) + for i, m := range p.moldings { + kinds[i] = m.Kind() + } + return kinds +} + +func (p *Planner) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind) error { + return p.enricher.EnrichStatus(ctx, kind, p.config) +} + +func (p *Planner) Mold(ctx context.Context, kind v1alpha1.MoldingKind) error { + for _, m := range p.moldings { + if m.Kind() == kind { + return m.MoldV1Alpha1(ctx, p.config) + } + } + return foundryerrors.Newf(foundryerrors.TypeInternal, "molding %q not registered for installation planner", kind) +} + +func (p *Planner) MergeStatusIntoSpec() error { + return p.config.MergeStatusIntoSpec() +} + +func (p *Planner) Forge(ctx context.Context, target string) ([]domain.Material, error) { + return p.casting.Forge(ctx, *p.config, target) +} + +func (p *Planner) Cast(ctx context.Context, poursPath string) error { + return p.casting.Cast(ctx, *p.config, poursPath) +} + +func (p *Planner) Toolers() []tooler.Tooler { + toolers := p.toolers + if p.config.Spec.Infrastructure.Enabled { + toolers = append(toolers, terraformtooler.New()) + } + return toolers +} diff --git a/internal/casting/installation/registry.go b/internal/casting/installation/registry.go new file mode 100644 index 0000000..a8e9aaa --- /dev/null +++ b/internal/casting/installation/registry.go @@ -0,0 +1,144 @@ +package installation + +import ( + "log/slog" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/internal/casting" + "github.com/signoz/foundry/internal/casting/coolifycasting" + "github.com/signoz/foundry/internal/casting/dockercomposecasting" + "github.com/signoz/foundry/internal/casting/dockerswarmcasting" + "github.com/signoz/foundry/internal/casting/ecsterraformcasting" + "github.com/signoz/foundry/internal/casting/kuberneteshelmcasting" + "github.com/signoz/foundry/internal/casting/kuberneteskustomizecasting" + "github.com/signoz/foundry/internal/casting/railwaytemplatecasting" + "github.com/signoz/foundry/internal/casting/rendercasting" + "github.com/signoz/foundry/internal/casting/systemdcasting" + foundryerrors "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/tooler" + "github.com/signoz/foundry/internal/tooler/clickhousekeepertooler" + "github.com/signoz/foundry/internal/tooler/clickhousetooler" + "github.com/signoz/foundry/internal/tooler/dockercomposetooler" + "github.com/signoz/foundry/internal/tooler/dockerswarmtooler" + "github.com/signoz/foundry/internal/tooler/dockertooler" + "github.com/signoz/foundry/internal/tooler/helmtooler" + "github.com/signoz/foundry/internal/tooler/kubectltooler" + "github.com/signoz/foundry/internal/tooler/postgrestooler" + "github.com/signoz/foundry/internal/tooler/systemdtooler" + "github.com/signoz/foundry/internal/tooler/terraformtooler" +) + +// Defines a single casting item in the registry. +type CastingItem struct { + // The particular casting implementation. + Casting casting.Casting + + // The toolers for the particular casting. + Toolers []tooler.Tooler +} + +type Registry struct { + // Castings for the different deployments. + castings map[v1alpha1.TypeDeployment]CastingItem +} + +func NewRegistry(logger *slog.Logger) *Registry { + return &Registry{ + castings: map[v1alpha1.TypeDeployment]CastingItem{ + { + Mode: v1alpha1.ModeDocker, + Flavor: v1alpha1.FlavorCompose, + }: { + Casting: dockercomposecasting.New(logger), + Toolers: []tooler.Tooler{dockertooler.New(), dockercomposetooler.New()}, + }, + { + Mode: v1alpha1.ModeSystemd, + Flavor: v1alpha1.FlavorBinary, + }: { + Casting: systemdcasting.New(logger), + Toolers: []tooler.Tooler{systemdtooler.New(), clickhousekeepertooler.New(), clickhousetooler.New(), postgrestooler.New()}, + }, + { + Mode: v1alpha1.ModeDocker, + Flavor: v1alpha1.FlavorSwarm, + }: { + Casting: dockerswarmcasting.New(logger), + Toolers: []tooler.Tooler{dockertooler.New(), dockerswarmtooler.New()}, + }, + { + Mode: v1alpha1.ModeKubernetes, + Flavor: v1alpha1.FlavorKustomize, + }: { + Casting: kuberneteskustomizecasting.New(logger), + Toolers: []tooler.Tooler{kubectltooler.New()}, + }, + { + Platform: v1alpha1.PlatformRender, + Flavor: v1alpha1.FlavorBlueprint, + }: { + Casting: rendercasting.New(logger), + }, + { + Platform: v1alpha1.PlatformCoolify, + Flavor: v1alpha1.FlavorStack, + }: { + Casting: coolifycasting.New(logger), + }, + { + Platform: v1alpha1.PlatformRailway, + Flavor: v1alpha1.FlavorTemplate, + }: { + Casting: railwaytemplatecasting.New(logger), + }, + { + Platform: v1alpha1.PlatformECS, + Flavor: v1alpha1.FlavorTerraform, + Mode: v1alpha1.ModeEC2, + }: { + Casting: ecsterraformcasting.New(logger), + Toolers: []tooler.Tooler{terraformtooler.New()}, + }, + { + Mode: v1alpha1.ModeKubernetes, + Flavor: v1alpha1.FlavorHelm, + }: { + Casting: kuberneteshelmcasting.New(logger), + Toolers: []tooler.Tooler{helmtooler.New()}, + }, + }, + } +} + +func (registry *Registry) CastingItems() map[v1alpha1.TypeDeployment]CastingItem { + return registry.castings +} + +func (registry *Registry) lookup(deployment v1alpha1.TypeDeployment) (CastingItem, bool) { + if item, ok := registry.castings[deployment]; ok { + return item, true + } + // Fall back to matching without platform (platform may be set for infra generation + // but the casting itself is platform-agnostic, e.g. docker/compose on aws). + if deployment.Platform != (v1alpha1.Platform{}) { + item, ok := registry.castings[v1alpha1.TypeDeployment{Mode: deployment.Mode, Flavor: deployment.Flavor}] + return item, ok + } + return CastingItem{}, false +} + +func (registry *Registry) Casting(deployment v1alpha1.TypeDeployment) (casting.Casting, error) { + item, ok := registry.lookup(deployment) + if !ok { + return nil, foundryerrors.Newf(foundryerrors.TypeUnsupported, "deployment '%+v' is not supported, raise an issue at https://github.com/signoz/foundry/issues to request support for this deployment", deployment) + } + return item.Casting, nil +} + +func (registry *Registry) Toolers(deployment v1alpha1.TypeDeployment) ([]tooler.Tooler, error) { + item, ok := registry.lookup(deployment) + if !ok { + return nil, foundryerrors.Newf(foundryerrors.TypeUnsupported, "deployment '%+v' is not supported, raise an issue at https://github.com/signoz/foundry/issues to request support for this deployment", deployment) + } + return item.Toolers, nil +} diff --git a/internal/casting/kuberneteshelmcasting/casting.go b/internal/casting/kuberneteshelmcasting/casting.go new file mode 100644 index 0000000..b9a6666 --- /dev/null +++ b/internal/casting/kuberneteshelmcasting/casting.go @@ -0,0 +1,224 @@ +package kuberneteshelmcasting + +import ( + "bytes" + "context" + "fmt" + "log/slog" + "os" + "path/filepath" + "time" + + "github.com/signoz/foundry/api/v1alpha1/installation" + rootcasting "github.com/signoz/foundry/internal/casting" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/molding" + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/cli" + "helm.sh/helm/v3/pkg/getter" + "helm.sh/helm/v3/pkg/repo" + "sigs.k8s.io/yaml" +) + +const ( + helmChartRepoUrl = "https://charts.signoz.io" + helmChartRepoName = "signoz" + helmChart = "signoz/signoz" + helmDeployTimeout = 10 * time.Minute + + annotationChart = "foundry.signoz.io/kubernetes-helm-casting-chart" + annotationRepoURL = "foundry.signoz.io/kubernetes-helm-casting-repo-url" + annotationRepoName = "foundry.signoz.io/kubernetes-helm-casting-repo-name" + annotationForgeChart = "foundry.signoz.io/kubernetes-helm-casting-forge-chart" +) + +var _ rootcasting.Casting = (*helmCasting)(nil) + +type helmCasting struct { + logger *slog.Logger + casting *domain.Template +} + +func New(logger *slog.Logger) *helmCasting { + return &helmCasting{ + logger: logger, + casting: valuesYAMLTemplate, + } +} + +func (c *helmCasting) Enricher(ctx context.Context, config *installation.Casting) (molding.MoldingEnricher, error) { + return newHelmMoldingEnricher(config), nil +} + +func (c *helmCasting) Forge(ctx context.Context, config installation.Casting, poursPath string) ([]domain.Material, error) { + buf := bytes.NewBuffer(nil) + err := valuesYAMLTemplate.Execute(buf, config) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to execute values yaml template") + } + + valuesBytes := buf.Bytes() + + valuesMaterial, err := domain.NewYAMLMaterial(valuesBytes, filepath.Join(rootcasting.DeploymentDir, "values.yaml")) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to create values yaml material") + } + + return []domain.Material{valuesMaterial}, nil +} + +func (c *helmCasting) Cast(ctx context.Context, config installation.Casting, poursPath string) error { + + valuesFile := filepath.Join(poursPath, rootcasting.DeploymentDir, "values.yaml") + if _, err := os.Stat(valuesFile); os.IsNotExist(err) { + return errors.Newf(errors.TypeNotFound, "values.yaml does not exist at path %s, run 'forge' first", valuesFile) + } + + valuesBytes, err := os.ReadFile(valuesFile) + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to read values file") + } + + vals := map[string]any{} + if err := yaml.Unmarshal(valuesBytes, &vals); err != nil { + return errors.Wrapf(err, errors.TypeInvalidInput, "failed to parse values") + } + + settings := cli.New() + settings.SetNamespace(config.Metadata.Name) + + actionConfig := new(action.Configuration) + if err := actionConfig.Init(settings.RESTClientGetter(), config.Metadata.Name, os.Getenv("HELM_DRIVER"), func(format string, v ...any) { + c.logger.Debug(fmt.Sprintf(format, v...)) + }); err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to initialize helm action config") + } + + var chartRef string + if c.shouldForgeChart(&config) { + chartRef = filepath.Join(poursPath, rootcasting.DeploymentDir, "chart", "signoz") + if _, err := os.Stat(chartRef); os.IsNotExist(err) { + return errors.Newf(errors.TypeNotFound, "local chart not found at %s, run 'forge' first with %s annotation set to 'true'", chartRef, annotationForgeChart) + } + c.logger.InfoContext(ctx, "Installing from local chart", slog.String("path", chartRef)) + } else { + repoURL := helmChartRepoUrl + if config.Metadata.Annotations != nil { + if u := config.Metadata.Annotations[annotationRepoURL]; u != "" { + repoURL = u + } + } + + chartRef = helmChart + if config.Metadata.Annotations != nil { + if ch := config.Metadata.Annotations[annotationChart]; ch != "" { + chartRef = ch + } + } + + repoName := helmChartRepoName + if config.Metadata.Annotations != nil { + if ch := config.Metadata.Annotations[annotationRepoName]; ch != "" { + chartRef = ch + } + } + + c.logger.InfoContext(ctx, "Adding Helm repo", slog.String("name", repoName), slog.String("url", repoURL), slog.String("chart", chartRef)) + if err := addHelmRepo(settings, repoName, repoURL); err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to add helm repo") + } + } + + c.logger.InfoContext(ctx, "Deploying with Helm", + slog.String("release", config.Metadata.Name), + slog.String("chart", chartRef), + slog.String("namespace", config.Metadata.Name), + ) + + histClient := action.NewHistory(actionConfig) + histClient.Max = 1 + _, err = histClient.Run(config.Metadata.Name) + + if err != nil { + install := action.NewInstall(actionConfig) + install.ReleaseName = config.Metadata.Name + install.Namespace = config.Metadata.Name + install.CreateNamespace = true + install.Wait = true + install.Timeout = helmDeployTimeout + + chartPath, err := install.LocateChart(chartRef, settings) + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to locate chart") + } + + chart, err := loader.Load(chartPath) + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to load chart") + } + + if _, err := install.RunWithContext(ctx, chart, vals); err != nil { + return errors.Wrapf(err, errors.TypeInternal, "helm install failed") + } + } else { + upgrade := action.NewUpgrade(actionConfig) + upgrade.Namespace = config.Metadata.Name + upgrade.Wait = true + upgrade.Timeout = helmDeployTimeout + + chartPath, err := upgrade.LocateChart(chartRef, settings) + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to locate chart") + } + + chart, err := loader.Load(chartPath) + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to load chart") + } + + if _, err := upgrade.RunWithContext(ctx, config.Metadata.Name, chart, vals); err != nil { + return errors.Wrapf(err, errors.TypeInternal, "helm upgrade failed") + } + } + + c.logger.InfoContext(ctx, "Helm deployment complete", + slog.String("release", config.Metadata.Name), + slog.String("namespace", config.Metadata.Name), + ) + return nil +} + +func (c *helmCasting) shouldForgeChart(config *installation.Casting) bool { + if config.Metadata.Annotations == nil { + return false + } + return config.Metadata.Annotations[annotationForgeChart] == "true" +} + +func addHelmRepo(settings *cli.EnvSettings, name, url string) error { + repoFile := settings.RepositoryConfig + repoEntry := &repo.Entry{ + Name: name, + URL: url, + } + + r, err := repo.NewChartRepository(repoEntry, getter.All(settings)) + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to create chart repository") + } + + r.CachePath = settings.RepositoryCache + if _, err := r.DownloadIndexFile(); err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to download repo index") + } + + f, err := repo.LoadFile(repoFile) + if err != nil { + f = repo.NewFile() + } + + f.Update(repoEntry) + return f.WriteFile(repoFile, 0644) +} diff --git a/internal/casting/kuberneteshelmcasting/embed.go b/internal/casting/kuberneteshelmcasting/embed.go new file mode 100644 index 0000000..6e1e67c --- /dev/null +++ b/internal/casting/kuberneteshelmcasting/embed.go @@ -0,0 +1,12 @@ +package kuberneteshelmcasting + +import ( + "embed" + + "github.com/signoz/foundry/internal/domain" +) + +//go:embed templates/values.yaml.gotmpl +var templates embed.FS + +var valuesYAMLTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/values.yaml.gotmpl", domain.FormatYAML) diff --git a/internal/casting/kuberneteshelmcasting/enricher.go b/internal/casting/kuberneteshelmcasting/enricher.go new file mode 100644 index 0000000..1b77448 --- /dev/null +++ b/internal/casting/kuberneteshelmcasting/enricher.go @@ -0,0 +1,84 @@ +package kuberneteshelmcasting + +import ( + "context" + "fmt" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/installation" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/molding" +) + +var _ molding.MoldingEnricher = (*helmMoldingEnricher)(nil) + +type helmMoldingEnricher struct { + materials []domain.Material +} + +func newHelmMoldingEnricher(_ *installation.Casting) *helmMoldingEnricher { + return &helmMoldingEnricher{materials: []domain.Material{}} +} + +func (e *helmMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *installation.Casting) error { + switch kind { + case v1alpha1.MoldingKindTelemetryStore: + return e.enrichTelemetryStore(config) + case v1alpha1.MoldingKindTelemetryKeeper: + return e.enrichTelemetryKeeper(config) + case v1alpha1.MoldingKindMetaStore: + return e.enrichMetaStore(config) + case v1alpha1.MoldingKindSignoz: + return e.enrichSignoz(config) + case v1alpha1.MoldingKindIngester: + return e.enrichIngester(config) + } + return nil +} + +func (e *helmMoldingEnricher) enrichTelemetryStore(config *installation.Casting) error { + name := fmt.Sprintf("%s-telemetrystore-%s", config.Metadata.Name, config.Spec.TelemetryStore.Kind) + config.Spec.TelemetryStore.Status.Addresses.TCP = []string{domain.MustNewAddress("tcp", name, 9000).String()} + return nil +} + +func (e *helmMoldingEnricher) enrichTelemetryKeeper(config *installation.Casting) error { + spec := &config.Spec.TelemetryKeeper + replicas := 1 + if spec.Spec.Cluster.Replicas != nil && *spec.Spec.Cluster.Replicas > 0 { + replicas = *spec.Spec.Cluster.Replicas + } + if replicas < 1 { + replicas = 1 + } + // Hardcoded to "zookeeper" because the chart deploys zookeeper, not clickhousekeeper. + base := fmt.Sprintf("%s-telemetrykeeper-zookeeper", config.Metadata.Name) + var client, raft []string + for i := 0; i < replicas; i++ { + client = append(client, domain.MustNewAddress("tcp", fmt.Sprintf("%s-%d", base, i), 9181).String()) + raft = append(raft, domain.MustNewAddress("tcp", fmt.Sprintf("%s-%d", base, i), 9234).String()) + } + config.Spec.TelemetryKeeper.Status.Addresses.Client = client + config.Spec.TelemetryKeeper.Status.Addresses.Raft = raft + return nil +} + +func (e *helmMoldingEnricher) enrichMetaStore(config *installation.Casting) error { + name := fmt.Sprintf("%s-metastore-%s", config.Metadata.Name, config.Spec.MetaStore.Kind) + config.Spec.MetaStore.Status.Addresses.DSN = []string{ + fmt.Sprintf("postgres://%s:5432", name), + } + return nil +} + +func (e *helmMoldingEnricher) enrichSignoz(config *installation.Casting) error { + // Chart uses signoz.fullname which resolves to fullnameOverride directly. + name := config.Metadata.Name + config.Spec.Signoz.Status.Addresses.Opamp = []string{domain.MustNewAddress("tcp", name, 4320).String()} + return nil +} + +func (e *helmMoldingEnricher) enrichIngester(config *installation.Casting) error { + // No-op: ingester molding only writes Status.Config.Data from other status. + return nil +} diff --git a/internal/casting/kuberneteshelmcasting/templates/values.yaml.gotmpl b/internal/casting/kuberneteshelmcasting/templates/values.yaml.gotmpl new file mode 100644 index 0000000..b2ab3d6 --- /dev/null +++ b/internal/casting/kuberneteshelmcasting/templates/values.yaml.gotmpl @@ -0,0 +1,107 @@ +fullnameOverride: {{ $.Metadata.Name }} +clickhouse: + enabled: {{ $.Spec.TelemetryStore.Spec.Enabled }} + fullnameOverride: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }} + {{- with $.Spec.TelemetryStore.Spec.Image }} + {{- $parts := splitn ":" 2 . }} + image: + repository: {{ $parts._0 }} + tag: {{ $parts._1 | default "latest" }} + {{- end }} + {{- if or $.Spec.TelemetryStore.Spec.Cluster.Shards $.Spec.TelemetryStore.Spec.Cluster.Replicas }} + layout: + shardsCount: {{ $.Spec.TelemetryStore.Spec.Cluster.Shards }} + replicasCount: {{ $.Spec.TelemetryStore.Spec.Cluster.Replicas }} + {{- end }} + zookeeper: + enabled: {{ $.Spec.TelemetryKeeper.Spec.Enabled }} + fullnameOverride: {{ $.Metadata.Name }}-telemetrykeeper-zookeeper + {{- with $.Spec.TelemetryKeeper.Spec.Image }} + {{- $parts := splitn ":" 2 . }} + image: + {{/* Currently charts does not support clickhousekeeper, we will render the keeper image here once we support clickhousekeeper in charts */}} + repository: signoz/zookeeper + tag: 3.7.1 + {{- end }} + replicaCount: {{ $.Spec.TelemetryKeeper.Spec.Cluster.Replicas }} +signoz: + name: signoz + {{- with $.Spec.Signoz.Spec.Image }} + {{- $parts := splitn ":" 2 . }} + image: + repository: {{ $parts._0 }} + tag: {{ $parts._1 | default "latest" }} + {{- end }} + {{- with $.Spec.Signoz.Spec.Cluster.Replicas }} + replicaCount: {{ . }} + {{- end }} + service: + type: ClusterIP + port: 8080 + internalPort: 8085 + opampPort: 4320 + {{- if $.Spec.Signoz.Spec.Env }} + env: + {{- with $.Spec.Signoz.Spec.Env }} +{{ toYaml . | indent 4 }} + {{- end }} + {{- end }} +otelCollector: + enabled: {{ $.Spec.Ingester.Spec.Enabled }} + name: ingester + {{- with $.Spec.Ingester.Spec.Image }} + {{- $parts := splitn ":" 2 . }} + image: + repository: {{ $parts._0 }} + tag: {{ $parts._1 | default "latest" }} + {{- end }} + {{- with $.Spec.Ingester.Spec.Cluster.Replicas }} + replicaCount: {{ . }} + {{- end }} + ports: + otlp: + enabled: true + containerPort: 4317 + servicePort: 4317 + protocol: TCP + otlp-http: + enabled: true + containerPort: 4318 + servicePort: 4318 + protocol: TCP + {{- if $.Spec.Ingester.Spec.Env }} + env: + {{- with $.Spec.Ingester.Spec.Env }} +{{ toYaml . | indent 4 }} + {{- end }} + {{- end }} +postgresql: + enabled: {{ $.Spec.MetaStore.Spec.Enabled }} + fullnameOverride: {{ $.Metadata.Name }}-metastore-{{ $.Spec.MetaStore.Kind }} + {{- with $.Spec.MetaStore.Spec.Image }} + {{- $parts := splitn ":" 2 . }} + image: + repository: {{ $parts._0 }} + tag: {{ $parts._1 | default "latest" }} + {{- end }} + {{- if $.Spec.MetaStore.Spec.Env }} + auth: + {{- with index $.Spec.MetaStore.Spec.Env "POSTGRES_DB" }} + database: {{ . }} + {{- end }} + {{- with index $.Spec.MetaStore.Spec.Env "POSTGRES_USER" }} + username: {{ . }} + {{- end }} + {{- with index $.Spec.MetaStore.Spec.Env "POSTGRES_PASSWORD" }} + password: {{ . }} + {{- end }} + {{- end }} +telemetryStoreMigrator: + enabled: true + name: {{ $.Metadata.Name }}-telemetrystore-migrator + {{- if $.Spec.Ingester.Spec.Env }} + env: + {{- with $.Spec.Ingester.Spec.Env }} +{{ toYaml . | indent 4 }} + {{- end }} + {{- end }} diff --git a/internal/casting/kuberneteskustomizecasting/casting.go b/internal/casting/kuberneteskustomizecasting/casting.go new file mode 100644 index 0000000..8ddd09b --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/casting.go @@ -0,0 +1,188 @@ +package kuberneteskustomizecasting + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/signoz/foundry/api/v1alpha1/installation" + rootcasting "github.com/signoz/foundry/internal/casting" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/molding" +) + +var _ rootcasting.Casting = (*kustomizeCasting)(nil) + +type kustomizeCasting struct { + logger *slog.Logger + castings []*domain.Template +} + +func New(logger *slog.Logger) *kustomizeCasting { + return &kustomizeCasting{ + logger: logger, + castings: []*domain.Template{ + clickhouseOperatorClusterrole, + clickhouseOperatorClusterrolebinding, + clickhouseOperatorConfigmap, + clickhouseOperatorDeployment, + clickhouseOperatorService, + clickhouseOperatorServiceaccount, + clickhouseInstanceInstallation, + clickhouseInstanceConfigmap, + clickhouseKeeperInstallation, + signozService, + signozServiceaccount, + signozStatefulset, + ingesterConfigmap, + ingesterDeployment, + ingesterService, + ingesterServiceaccount, + metastoreService, + metastoreServiceaccount, + metastoreStatefulset, + telemetrystoreMigratorJob, + clickhouseOperatorKustomization, + clickhouseInstallationKustomization, + clickhouseKeeperKustomization, + signozKustomization, + ingesterKustomization, + metastoreKustomization, + telemetrystoreMigratorKustomization, + deploymentNamespace, + deploymentKustomization, + }, + } +} + +func (c *kustomizeCasting) Enricher(ctx context.Context, config *installation.Casting) (molding.MoldingEnricher, error) { + return newKustomizeMoldingEnricher(config) +} + +func (c *kustomizeCasting) Forge(ctx context.Context, cfg installation.Casting, poursPath string) ([]domain.Material, error) { + var materials []domain.Material + for _, tmpl := range c.castings { + m, err := c.forgeCasting(tmpl, &cfg, poursPath) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to forge") + } + materials = append(materials, m...) + } + return materials, nil +} + +const clickhouseOperatorVersion = "0.25.3" + +var clickhouseCRDs = []string{ + "clickhouseinstallations.clickhouse.altinity.com.crd.yaml", + "clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml", + "clickhouseoperatorconfigurations.clickhouse.altinity.com.crd.yaml", + "clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml", +} + +func (c *kustomizeCasting) Cast(ctx context.Context, config installation.Casting, poursPath string) error { + c.logger.InfoContext(ctx, "Applying kustomize manifests") + + kustomizeDir := filepath.Join(poursPath, rootcasting.DeploymentDir) + if _, err := os.Stat(filepath.Join(kustomizeDir, "kustomization.yaml")); os.IsNotExist(err) { + return errors.Newf(errors.TypeNotFound, "kustomization.yaml does not exist at path: %s, run 'forge' first", kustomizeDir) + } + + runctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + if err := c.applyCRDs(runctx); err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to apply CRDs") + } + + cmd := exec.CommandContext(runctx, "kubectl", "apply", "-k", kustomizeDir) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + c.logger.DebugContext(runctx, "Running command", + slog.String("command", fmt.Sprintf("kubectl apply -k %s", kustomizeDir))) + + if err := cmd.Run(); err != nil { + c.logger.ErrorContext(runctx, "kubectl apply failed", slog.String("error", err.Error())) + return errors.Wrapf(err, errors.TypeInternal, "kubectl apply -k failed") + } + + c.logger.InfoContext(runctx, "Kustomize manifests applied successfully") + return nil +} + +func (c *kustomizeCasting) applyCRDs(ctx context.Context) error { + c.logger.InfoContext(ctx, "Applying ClickHouse CRDs", + slog.String("version", clickhouseOperatorVersion)) + + for _, crd := range clickhouseCRDs { + url := fmt.Sprintf( + "https://raw.githubusercontent.com/Altinity/clickhouse-operator/%s/deploy/operatorhub/%s/%s", + clickhouseOperatorVersion, clickhouseOperatorVersion, crd, + ) + + cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", url) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + c.logger.DebugContext(ctx, "Applying CRD", slog.String("url", url)) + + if err := cmd.Run(); err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to apply CRD %s", crd) + } + } + + return nil +} + +func (c *kustomizeCasting) forgeCasting(tmpl *domain.Template, cfg *installation.Casting, poursPath string) ([]domain.Material, error) { + templatePath := tmpl.Path() + relPath := strings.TrimPrefix(templatePath, "templates/") + relPath = strings.TrimSuffix(relPath, filepath.Ext(relPath)) + path := filepath.Join(rootcasting.DeploymentDir, relPath) + material, err := tmpl.Render(cfg, path) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "render template %s", templatePath) + } + return []domain.Material{material}, nil +} + +func getOverrideMaterials(config *installation.Casting) ([]domain.StructuredMaterial, error) { + return renderStructured(config, []templateAt{ + {telemetryStoreOverrideTemplate, "store_overrides.yaml"}, + }) +} + +func getServiceMaterials(config *installation.Casting) ([]domain.StructuredMaterial, error) { + return renderStructured(config, []templateAt{ + {clickhouseInstanceInstallation, "clickhouseInstallation.yaml"}, + {metastoreService, "metastoreServie.yaml"}, + }) +} + +type templateAt struct { + tmpl *domain.Template + path string +} + +func renderStructured(config *installation.Casting, items []templateAt) ([]domain.StructuredMaterial, error) { + materials := make([]domain.StructuredMaterial, 0, len(items)) + for _, item := range items { + m, err := item.tmpl.Render(config, item.path) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "render template %s", item.tmpl.Path()) + } + sm, ok := m.(domain.StructuredMaterial) + if !ok { + return nil, errors.Newf(errors.TypeInternal, "template %s does not produce a structured material", item.tmpl.Path()) + } + materials = append(materials, sm) + } + return materials, nil +} diff --git a/internal/casting/kuberneteskustomizecasting/embed.go b/internal/casting/kuberneteskustomizecasting/embed.go new file mode 100644 index 0000000..98683c4 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/embed.go @@ -0,0 +1,60 @@ +package kuberneteskustomizecasting + +import ( + "embed" + + "github.com/signoz/foundry/internal/domain" +) + +//go:embed templates/*/*.gotmpl templates/*/*/*.gotmpl templates/*.gotmpl +var templates embed.FS + +var ( + // telemetrystore/clickhouse-operator. + clickhouseOperatorClusterrole = domain.MustNewTemplateFromFS(templates, "templates/clickhouse-operator/clusterrole.yaml.gotmpl", domain.FormatYAML) + clickhouseOperatorClusterrolebinding = domain.MustNewTemplateFromFS(templates, "templates/clickhouse-operator/clusterrolebinding.yaml.gotmpl", domain.FormatYAML) + clickhouseOperatorConfigmap = domain.MustNewTemplateFromFS(templates, "templates/clickhouse-operator/configmap.yaml.gotmpl", domain.FormatYAML) + clickhouseOperatorDeployment = domain.MustNewTemplateFromFS(templates, "templates/clickhouse-operator/deployment.yaml.gotmpl", domain.FormatYAML) + clickhouseOperatorService = domain.MustNewTemplateFromFS(templates, "templates/clickhouse-operator/service.yaml.gotmpl", domain.FormatYAML) + clickhouseOperatorServiceaccount = domain.MustNewTemplateFromFS(templates, "templates/clickhouse-operator/serviceaccount.yaml.gotmpl", domain.FormatYAML) + clickhouseOperatorKustomization = domain.MustNewTemplateFromFS(templates, "templates/clickhouse-operator/kustomization.yaml.gotmpl", domain.FormatYAML) + + // telemetrystore/clickhouse. + clickhouseInstanceInstallation = domain.MustNewTemplateFromFS(templates, "templates/telemetrystore/clickhouse/clickhouseinstallation.yaml.gotmpl", domain.FormatYAML) + clickhouseInstanceConfigmap = domain.MustNewTemplateFromFS(templates, "templates/telemetrystore/clickhouse/configmap.yaml.gotmpl", domain.FormatYAML) + clickhouseInstallationKustomization = domain.MustNewTemplateFromFS(templates, "templates/telemetrystore/clickhouse/kustomization.yaml.gotmpl", domain.FormatYAML) + + // telemetrykeeper. + clickhouseKeeperInstallation = domain.MustNewTemplateFromFS(templates, "templates/telemetrykeeper/clickhousekeeper/clickhousekeeperinstallation.yaml.gotmpl", domain.FormatYAML) + clickhouseKeeperKustomization = domain.MustNewTemplateFromFS(templates, "templates/telemetrykeeper/clickhousekeeper/kustomization.yaml.gotmpl", domain.FormatYAML) + + // signoz. + signozService = domain.MustNewTemplateFromFS(templates, "templates/signoz/service.yaml.gotmpl", domain.FormatYAML) + signozServiceaccount = domain.MustNewTemplateFromFS(templates, "templates/signoz/serviceaccount.yaml.gotmpl", domain.FormatYAML) + signozStatefulset = domain.MustNewTemplateFromFS(templates, "templates/signoz/statefulset.yaml.gotmpl", domain.FormatYAML) + signozKustomization = domain.MustNewTemplateFromFS(templates, "templates/signoz/kustomization.yaml.gotmpl", domain.FormatYAML) + + // ingester. + ingesterConfigmap = domain.MustNewTemplateFromFS(templates, "templates/ingester/configmap.yaml.gotmpl", domain.FormatYAML) + ingesterDeployment = domain.MustNewTemplateFromFS(templates, "templates/ingester/deployment.yaml.gotmpl", domain.FormatYAML) + ingesterService = domain.MustNewTemplateFromFS(templates, "templates/ingester/service.yaml.gotmpl", domain.FormatYAML) + ingesterServiceaccount = domain.MustNewTemplateFromFS(templates, "templates/ingester/serviceaccount.yaml.gotmpl", domain.FormatYAML) + ingesterKustomization = domain.MustNewTemplateFromFS(templates, "templates/ingester/kustomization.yaml.gotmpl", domain.FormatYAML) + + // metastore/postgres. + metastoreService = domain.MustNewTemplateFromFS(templates, "templates/metastore/postgres/service.yaml.gotmpl", domain.FormatYAML) + metastoreServiceaccount = domain.MustNewTemplateFromFS(templates, "templates/metastore/postgres/serviceaccount.yaml.gotmpl", domain.FormatYAML) + metastoreStatefulset = domain.MustNewTemplateFromFS(templates, "templates/metastore/postgres/statefulset.yaml.gotmpl", domain.FormatYAML) + metastoreKustomization = domain.MustNewTemplateFromFS(templates, "templates/metastore/postgres/kustomization.yaml.gotmpl", domain.FormatYAML) + + // telemetrystore-migrator. + telemetrystoreMigratorJob = domain.MustNewTemplateFromFS(templates, "templates/telemetrystore-migrator/job.yaml.gotmpl", domain.FormatYAML) + telemetrystoreMigratorKustomization = domain.MustNewTemplateFromFS(templates, "templates/telemetrystore-migrator/kustomization.yaml.gotmpl", domain.FormatYAML) + + // deployment. + deploymentNamespace = domain.MustNewTemplateFromFS(templates, "templates/namespace.yaml.gotmpl", domain.FormatYAML) + deploymentKustomization = domain.MustNewTemplateFromFS(templates, "templates/kustomization.yaml.gotmpl", domain.FormatYAML) + + // molding overrides. + telemetryStoreOverrideTemplate = domain.MustNewTemplateFromFS(templates, "templates/overrides/telemetrystore.yaml.gotmpl", domain.FormatYAML) +) diff --git a/internal/casting/kuberneteskustomizecasting/enricher.go b/internal/casting/kuberneteskustomizecasting/enricher.go new file mode 100644 index 0000000..5bf2712 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/enricher.go @@ -0,0 +1,118 @@ +package kuberneteskustomizecasting + +import ( + "context" + "fmt" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/installation" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/molding" +) + +const ( + telemetryStorePort = 9000 + telemetryKeeperClientPort = 9181 + telemetryKeeperRaftPort = 9234 + signozOpampPort = 4320 +) + +var _ molding.MoldingEnricher = (*kustomizeMoldingEnricher)(nil) + +type kustomizeMoldingEnricher struct { + materials []domain.StructuredMaterial + overrideMaterials []domain.StructuredMaterial +} + +func newKustomizeMoldingEnricher(config *installation.Casting) (*kustomizeMoldingEnricher, error) { + materials, err := getServiceMaterials(config) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to get service yaml material") + } + + overrideMaterials, err := getOverrideMaterials(config) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to get override materials") + } + + return &kustomizeMoldingEnricher{ + materials: materials, + overrideMaterials: overrideMaterials, + }, nil +} + +func (e *kustomizeMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *installation.Casting) error { + switch kind { + case v1alpha1.MoldingKindTelemetryStore: + return e.enrichTelemetryStore(config) + case v1alpha1.MoldingKindTelemetryKeeper: + return e.enrichTelemetryKeeper(config) + case v1alpha1.MoldingKindMetaStore: + return e.enrichMetaStore(config) + case v1alpha1.MoldingKindSignoz: + return e.enrichSignoz(config) + case v1alpha1.MoldingKindIngester: + return e.enrichIngester(config) + } + return nil +} + +func (e *kustomizeMoldingEnricher) enrichTelemetryStore(config *installation.Casting) error { + name, err := e.materials[0].GetBytes("spec.templates.serviceTemplates.0.generateName") + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to get telemetrystore service names") + } + config.Spec.TelemetryStore.Status.Addresses.TCP = []string{domain.MustNewAddress("tcp", string(name), telemetryStorePort).String()} + + if config.Spec.TelemetryStore.Status.Extras == nil { + config.Spec.TelemetryStore.Status.Extras = make(map[string]string) + } + config.Spec.TelemetryStore.Status.Extras["_overrides"] = string(e.overrideMaterials[0].FmtContents()) + + return nil +} + +func (e *kustomizeMoldingEnricher) enrichTelemetryKeeper(config *installation.Casting) error { + spec := &config.Spec.TelemetryKeeper + replicas := 1 + if spec.Spec.Cluster.Replicas != nil && *spec.Spec.Cluster.Replicas > 0 { + replicas = *spec.Spec.Cluster.Replicas + } + if replicas < 1 { + replicas = 1 + } + // Dummy Variables, To pass validation in molding + // TODO: Take the logic out of molding as operator handles it already + base := config.Metadata.Name + "-clickhouse-keeper" + var client, raft []string + for i := 0; i < replicas; i++ { + client = append(client, domain.MustNewAddress("tcp", fmt.Sprintf("%s-%d", base, i), telemetryKeeperClientPort).String()) + raft = append(raft, domain.MustNewAddress("tcp", fmt.Sprintf("%s-%d", base, i), telemetryKeeperRaftPort).String()) + } + config.Spec.TelemetryKeeper.Status.Addresses.Client = client + config.Spec.TelemetryKeeper.Status.Addresses.Raft = raft + return nil +} + +func (e *kustomizeMoldingEnricher) enrichMetaStore(config *installation.Casting) error { + name, err := e.materials[1].GetBytes("metadata.name") + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to get metastore service names") + } + config.Spec.MetaStore.Status.Addresses.DSN = []string{ + fmt.Sprintf("postgres://%s:5432", name), + } + return nil +} + +func (e *kustomizeMoldingEnricher) enrichSignoz(config *installation.Casting) error { + name := config.Metadata.Name + "-signoz" + config.Spec.Signoz.Status.Addresses.Opamp = []string{domain.MustNewAddress("ws", name, signozOpampPort).String()} + return nil +} + +func (e *kustomizeMoldingEnricher) enrichIngester(config *installation.Casting) error { + // No-op: ingester molding only writes Status.Config.Data from other status. + return nil +} diff --git a/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/clusterrole.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/clusterrole.yaml.gotmpl new file mode 100644 index 0000000..66c7449 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/clusterrole.yaml.gotmpl @@ -0,0 +1,203 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + labels: + app.kubernetes.io/name: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + app.kubernetes.io/instance: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + app.kubernetes.io/version: "0.25.3" +rules: + # + # Core API group + # + - apiGroups: + - "" + resources: + - configmaps + - services + - persistentvolumeclaims + - secrets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch + - delete + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + # + # apps.* resources + # + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + - apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - patch + - update + - delete + # The operator deployment personally, identified by name + - apiGroups: + - apps + resources: + - deployments + resourceNames: + - clickhouse-operator + verbs: + - get + - patch + - update + - delete + # + # policy.* resources + # + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + # + # apiextensions + # + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + # clickhouse - related resources + - apiGroups: + - clickhouse.altinity.com + # + # The operators specific Custom Resources + # + + resources: + - clickhouseinstallations + verbs: + - get + - list + - watch + - patch + - update + - delete + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallationtemplates + - clickhouseoperatorconfigurations + verbs: + - get + - list + - watch + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/finalizers + - clickhouseinstallationtemplates/finalizers + - clickhouseoperatorconfigurations/finalizers + verbs: + - update + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/status + - clickhouseinstallationtemplates/status + - clickhouseoperatorconfigurations/status + verbs: + - get + - update + - patch + - create + - delete + # clickhouse-keeper - related resources + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations + verbs: + - get + - list + - watch + - patch + - update + - delete + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/finalizers + verbs: + - update + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/status + verbs: + - get + - update + - patch + - create + - delete diff --git a/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/clusterrolebinding.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/clusterrolebinding.yaml.gotmpl new file mode 100644 index 0000000..37fb7fb --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/clusterrolebinding.yaml.gotmpl @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + labels: + app.kubernetes.io/name: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + app.kubernetes.io/instance: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + app.kubernetes.io/version: "0.25.3" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator +subjects: + - kind: ServiceAccount + name: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + namespace: {{ $.Metadata.Name }} diff --git a/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/configmap.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/configmap.yaml.gotmpl new file mode 100644 index 0000000..a097650 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/configmap.yaml.gotmpl @@ -0,0 +1,580 @@ +{{- $name := print $.Metadata.Name "-telemetrystore-" $.Spec.TelemetryStore.Kind "-operator" -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-etc-confd-files + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $name }} + app.kubernetes.io/version: "0.25.3" +data: + null +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-etc-configd-files + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $name }} + app.kubernetes.io/version: "0.25.3" +data: + 01-clickhouse-01-listen.xml: | + <clickhouse> + <!-- Listen wildcard address to allow accepting connections from other containers and host network. --> + <listen_host>::</listen_host> + <listen_host>0.0.0.0</listen_host> + <listen_try>1</listen_try> + </clickhouse> + 01-clickhouse-02-logger.xml: | + <clickhouse> + <logger> + <!-- Possible levels: https://github.com/pocoproject/poco/blob/devel/Foundation/include/Poco/Logger.h#L105 --> + <level>warning</level> + <formatting> + <type>json</type> + </formatting> + <log>/var/log/clickhouse-server/clickhouse-server.log</log> + <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog> + <size>1000M</size> + <count>10</count> + <!-- Default behavior is autodetection (log to console if not daemon mode and is tty) --> + <console>1</console> + </logger> + </clickhouse> + 01-clickhouse-03-query_log.xml: | + <clickhouse> + <query_log replace="1"> + <database>system</database> + <table>query_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </query_log> + </clickhouse> + 01-clickhouse-04-part_log.xml: | + <clickhouse> + <part_log replace="1"> + <database>system</database> + <table>part_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </part_log> + </clickhouse> + 01-clickhouse-05-trace_log.xml: |- + <clickhouse> + <trace_log replace="1"> + <database>system</database> + <table>trace_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </trace_log> + </clickhouse> + 01-clickhouse-06-asynchronous_insert_log.xml: | + <clickhouse> + <asynchronous_insert_log replace="1"> + <database>system</database> + <table>asynchronous_insert_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </asynchronous_insert_log> + </clickhouse> + 01-clickhouse-07-asynchronous_metric_log.xml: | + <clickhouse> + <asynchronous_metric_log replace="1"> + <database>system</database> + <table>asynchronous_metric_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </asynchronous_metric_log> + </clickhouse> + 01-clickhouse-08-backup_log.xml: | + <clickhouse> + <backup_log replace="1"> + <database>system</database> + <table>backup_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </backup_log> + </clickhouse> + 01-clickhouse-09-blob_storage_log.xml: | + <clickhouse> + <blob_storage_log replace="1"> + <database>system</database> + <table>blob_storage_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </blob_storage_log> + </clickhouse> + 01-clickhouse-10-crash_log.xml: | + <clickhouse> + <crash_log replace="1"> + <database>system</database> + <table>crash_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 14 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </crash_log> + </clickhouse> + 01-clickhouse-11-metric_log.xml: | + <clickhouse> + <metric_log replace="1"> + <database>system</database> + <table>metric_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </metric_log> + </clickhouse> + 01-clickhouse-12-query_thread_log.xml: | + <clickhouse> + <query_thread_log replace="1"> + <database>system</database> + <table>query_thread_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </query_thread_log> + </clickhouse> + 01-clickhouse-13-query_views_log.xml: | + <clickhouse> + <query_views_log replace="1"> + <database>system</database> + <table>query_views_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </query_views_log> + </clickhouse> + 01-clickhouse-14-session_log.xml: | + <clickhouse> + <session_log replace="1"> + <database>system</database> + <table>session_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </session_log> + </clickhouse> + 01-clickhouse-15-zookeeper_log.xml: | + <clickhouse> + <zookeeper_log replace="1"> + <database>system</database> + <table>zookeeper_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 3 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </zookeeper_log> + </clickhouse> + 01-clickhouse-16-processors_profile_log.xml: | + <clickhouse> + <processors_profile_log replace="1"> + <database>system</database> + <table>processors_profile_log</table> + <engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 7 day</engine> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </processors_profile_log> + </clickhouse> +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-etc-files + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $name }} + app.kubernetes.io/version: "0.25.3" +data: + config.yaml: |- + annotation: + exclude: + - argocd.argoproj.io/tracking-id + include: [] + clickhouse: + access: + password: "" + port: 8123 + rootCA: "" + scheme: auto + secret: + name: "clickhouse-operator" + namespace: "{{ $.Metadata.Name }}" + timeouts: + connect: 1 + query: 4 + username: "" + addons: + rules: + - spec: + configuration: + files: null + profiles: null + quotas: null + settings: null + users: null + version: '*' + - spec: + configuration: + files: null + profiles: null + quotas: null + settings: null + users: + '{clickhouseOperatorUser}/access_management': 1 + '{clickhouseOperatorUser}/named_collection_control': 1 + '{clickhouseOperatorUser}/show_named_collections': 1 + '{clickhouseOperatorUser}/show_named_collections_secrets': 1 + version: '>= 23.3' + - spec: + configuration: + files: null + profiles: + clickhouse_operator/format_display_secrets_in_show_and_select: 1 + quotas: null + settings: + display_secrets_in_show_and_select: 1 + users: null + version: '>= 23.5' + configuration: + file: + path: + common: chi/config.d + host: chi/conf.d + user: chi/users.d + network: + hostRegexpTemplate: (chi-{chi}-[^.]+\d+-\d+|clickhouse\-{chi})\.{namespace}\.svc\.cluster\.local$ + user: + default: + networksIP: + - ::1 + - 127.0.0.1 + password: default + profile: default + quota: default + configurationRestartPolicy: + rules: + - rules: + - settings/*: "yes" + - settings/access_control_path: "no" + - settings/dictionaries_config: "no" + - settings/max_server_memory_*: "no" + - settings/max_*_to_drop: "no" + - settings/max_concurrent_queries: "no" + - settings/models_config: "no" + - settings/user_defined_executable_functions_config: "no" + - settings/logger/*: "no" + - settings/macros/*: "no" + - settings/remote_servers/*: "no" + - settings/user_directories/*: "no" + - settings/display_secrets_in_show_and_select: "no" + - zookeeper/*: "yes" + - files/*.xml: "yes" + - files/config.d/*.xml: "yes" + - files/config.d/*dict*.xml: "no" + - files/config.d/*no_restart*: "no" + - profiles/default/background_*_pool_size: "yes" + - profiles/default/max_*_for_server: "yes" + version: '*' + - rules: + - settings/logger: "yes" + version: 21.* + metrics: + timeouts: + collect: 9 + keeper: + configuration: + file: + path: + common: chk/keeper_config.d + host: chk/conf.d + user: chk/users.d + label: + appendScope: "no" + exclude: [] + include: [] + logger: + alsologtostderr: "false" + log_backtrace_at: "" + logtostderr: "true" + stderrthreshold: "" + v: "1" + vmodule: "" + metrics: + labels: + exclude: [] + pod: + terminationGracePeriod: 30 + reconcile: + host: + wait: + exclude: true + include: false + queries: true + replicas: + all: false + delay: 10 + new: true + runtime: + reconcileCHIsThreadsNumber: 10 + reconcileShardsMaxConcurrencyPercent: 50 + reconcileShardsThreadsNumber: 5 + statefulSet: + create: + onFailure: ignore + update: + onFailure: abort + pollInterval: 5 + timeout: 300 + statefulSet: + revisionHistoryLimit: 0 + status: + fields: + action: false + actions: false + error: false + errors: false + template: + chi: + path: chi/templates.d + policy: ApplyOnNextReconcile + chk: + path: chk/templates.d + policy: ApplyOnNextReconcile + watch: + namespaces: + - ".*" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-etc-templatesd-files + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $name }} + app.kubernetes.io/version: "0.25.3" +data: + 001-templates.json.example: | + { + "apiVersion": "clickhouse.altinity.com/v1", + "kind": "ClickHouseInstallationTemplate", + "metadata": { + "name": "01-default-volumeclaimtemplate" + }, + "spec": { + "templates": { + "volumeClaimTemplates": [ + { + "name": "chi-default-volume-claim-template", + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "2Gi" + } + } + } + } + ], + "podTemplates": [ + { + "name": "chi-default-oneperhost-pod-template", + "distribution": "OnePerHost", + "spec": { + "containers" : [ + { + "name": "clickhouse", + "image": "clickhouse/clickhouse-server:23.8", + "ports": [ + { + "name": "http", + "containerPort": 8123 + }, + { + "name": "client", + "containerPort": 9000 + }, + { + "name": "interserver", + "containerPort": 9009 + } + ] + } + ] + } + } + ] + } + } + } + default-pod-template.yaml.example: | + apiVersion: "clickhouse.altinity.com/v1" + kind: "ClickHouseInstallationTemplate" + metadata: + name: "default-oneperhost-pod-template" + spec: + templates: + podTemplates: + - name: default-oneperhost-pod-template + distribution: "OnePerHost" + default-storage-template.yaml.example: | + apiVersion: "clickhouse.altinity.com/v1" + kind: "ClickHouseInstallationTemplate" + metadata: + name: "default-storage-template-2Gi" + spec: + templates: + volumeClaimTemplates: + - name: default-storage-template-2Gi + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + readme: Templates in this folder are packaged with an operator and available via 'useTemplate' +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-etc-usersd-files + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $name }} + app.kubernetes.io/version: "0.25.3" +data: + 01-clickhouse-user.xml: | + <yandex> + <users> + <clickhouse_operator> + <networks> + <ip>127.0.0.1</ip> + </networks> + <password_sha256_hex></password_sha256_hex> + <profile>clickhouse_operator</profile> + <quota>default</quota> + </clickhouse_operator> + </users> + <profiles> + <clickhouse_operator> + <log_queries>0</log_queries> + <skip_unavailable_shards>1</skip_unavailable_shards> + <http_connection_timeout>10</http_connection_timeout> + </clickhouse_operator> + </profiles> + </yandex> + 02-clickhouse-default-profile.xml: | + <yandex> + <profiles> + <default> + <log_queries>1</log_queries> + <connect_timeout_with_failover_ms>1000</connect_timeout_with_failover_ms> + <distributed_aggregation_memory_efficient>1</distributed_aggregation_memory_efficient> + <parallel_view_processing>1</parallel_view_processing> + </default> + </profiles> + </yandex> + 03-database-ordinary.xml: | + <yandex> + <profiles> + <default> + <default_database_engine>Ordinary</default_database_engine> + </default> + </profiles> + </yandex> +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-keeper-etc-confd-files + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $name }} + app.kubernetes.io/version: "0.25.3" +data: + null +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-keeper-etc-configd-files + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $name }} + app.kubernetes.io/version: "0.25.3" +data: + 01-keeper-01-default-config.xml: | + <!-- IMPORTANT --> + <!-- This file is auto-generated --> + <!-- Do not edit this file - all changes would be lost --> + <!-- Edit appropriate template in the following folder: --> + <!-- deploy/builder/templates-config --> + <!-- IMPORTANT --> + <clickhouse> + <keeper_server> + <coordination_settings> + <min_session_timeout_ms>10000</min_session_timeout_ms> + <operation_timeout_ms>10000</operation_timeout_ms> + <raft_logs_level>information</raft_logs_level> + <session_timeout_ms>100000</session_timeout_ms> + </coordination_settings> + <hostname_checks_enabled>true</hostname_checks_enabled> + <log_storage_path>/var/lib/clickhouse-keeper/coordination/logs</log_storage_path> + <snapshot_storage_path>/var/lib/clickhouse-keeper/coordination/snapshots</snapshot_storage_path> + <storage_path>/var/lib/clickhouse-keeper</storage_path> + <tcp_port>2181</tcp_port> + </keeper_server> + <listen_host>::</listen_host> + <listen_host>0.0.0.0</listen_host> + <listen_try>1</listen_try> + <logger> + <console>1</console> + <level>information</level> + </logger> + <max_connections>4096</max_connections> + </clickhouse> + 01-keeper-02-readiness.xml: | + <!-- IMPORTANT --> + <!-- This file is auto-generated --> + <!-- Do not edit this file - all changes would be lost --> + <!-- Edit appropriate template in the following folder: --> + <!-- deploy/builder/templates-config --> + <!-- IMPORTANT --> + <clickhouse> + <keeper_server> + <http_control> + <port>9182</port> + <readiness> + <endpoint>/ready</endpoint> + </readiness> + </http_control> + </keeper_server> + </clickhouse> + 01-keeper-03-enable-reconfig.xml: |- + <!-- IMPORTANT --> + <!-- This file is auto-generated --> + <!-- Do not edit this file - all changes would be lost --> + <!-- Edit appropriate template in the following folder: --> + <!-- deploy/builder/templates-config --> + <!-- IMPORTANT --> + <clickhouse> + <keeper_server> + <enable_reconfiguration>false</enable_reconfiguration> + </keeper_server> + </clickhouse> +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-keeper-etc-templatesd-files + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $name }} + app.kubernetes.io/version: "0.25.3" +data: + readme: Templates in this folder are packaged with an operator and available via 'useTemplate' +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-keeper-etc-usersd-files + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $name }} + app.kubernetes.io/version: "0.25.3" +data: + null diff --git a/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/deployment.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/deployment.yaml.gotmpl new file mode 100644 index 0000000..195ffd8 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/deployment.yaml.gotmpl @@ -0,0 +1,230 @@ +{{- $name := print $.Metadata.Name "-telemetrystore-" $.Spec.TelemetryStore.Kind "-operator" -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + labels: + app.kubernetes.io/name: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + app.kubernetes.io/instance: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + app.kubernetes.io/version: "0.25.3" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + app.kubernetes.io/instance: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + template: + metadata: + labels: + app.kubernetes.io/name: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + app.kubernetes.io/instance: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + app.kubernetes.io/version: "0.25.3" + app.kubernetes.io/managed-by: kustomize + annotations: + clickhouse-operator-metrics/port: "9999" + clickhouse-operator-metrics/scrape: "true" + checksum/files: 420251c3f150460f0b1b15211b9c0b965c16342055c2f3a5a57768d229b2ddd0 + checksum/confd-files: 6a503b551565d380d3b4ea4b4c48a66629379558aa6cf0919d55f88f3cb4f56b + checksum/configd-files: a8b2e79da1b016cb2fa1e1b3d61f1edd8872af054a7419040c28b15a236bdc24 + checksum/templatesd-files: bfbe2e3230fe68164813fcaf74c5cc06b11878c26c764476399ac8803c8bde1c + checksum/usersd-files: c609c91ce94021945657aae781f199517cbc6f2c1f603238c88004f8acf84f95 + checksum/keeper-confd-files: c17fcdc4ce46db171e44a8e5036cafb7173a9806470fafdef717cc3c01318470 + checksum/keeper-configd-files: beed860cfbb0c9e2a0b266f65dee8cbaf22990dc3e200c93b74e697b3cf4c9df + checksum/keeper-templatesd-files: a47abf45edf0f439048648e3d6d1ecede35e560a76f32d8a4aa6055e0fcf58a8 + checksum/keeper-usersd-files: 2fbf34a572990f10fe9bf11c9884c5e847a1a23a02f0728e83bac4f4f9036797 + spec: + nodeSelector: + kubernetes.io/os: "linux" + serviceAccountName: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + volumes: + - name: etc-clickhouse-operator-folder + configMap: + name: {{ $name }}-etc-files + - name: etc-clickhouse-operator-confd-folder + configMap: + name: {{ $name }}-etc-confd-files + - name: etc-clickhouse-operator-configd-folder + configMap: + name: {{ $name }}-etc-configd-files + - name: etc-clickhouse-operator-templatesd-folder + configMap: + name: {{ $name }}-etc-templatesd-files + - name: etc-clickhouse-operator-usersd-folder + configMap: + name: {{ $name }}-etc-usersd-files + - name: etc-keeper-operator-confd-folder + configMap: + name: {{ $name }}-keeper-etc-confd-files + - name: etc-keeper-operator-configd-folder + configMap: + name: {{ $name }}-keeper-etc-configd-files + - name: etc-keeper-operator-templatesd-folder + configMap: + name: {{ $name }}-keeper-etc-templatesd-files + - name: etc-keeper-operator-usersd-folder + configMap: + name: {{ $name }}-keeper-etc-usersd-files + containers: + - name: operator + image: altinity/clickhouse-operator:0.25.3 + imagePullPolicy: IfNotPresent + volumeMounts: + - name: etc-clickhouse-operator-folder + mountPath: /etc/clickhouse-operator + - name: etc-clickhouse-operator-confd-folder + mountPath: /etc/clickhouse-operator/chi/conf.d + - name: etc-clickhouse-operator-configd-folder + mountPath: /etc/clickhouse-operator/chi/config.d + - name: etc-clickhouse-operator-templatesd-folder + mountPath: /etc/clickhouse-operator/chi/templates.d + - name: etc-clickhouse-operator-usersd-folder + mountPath: /etc/clickhouse-operator/chi/users.d + - name: etc-keeper-operator-confd-folder + mountPath: /etc/clickhouse-operator/chk/conf.d + - name: etc-keeper-operator-configd-folder + mountPath: /etc/clickhouse-operator/chk/keeper_config.d + - name: etc-keeper-operator-templatesd-folder + mountPath: /etc/clickhouse-operator/chk/templates.d + - name: etc-keeper-operator-usersd-folder + mountPath: /etc/clickhouse-operator/chk/users.d + env: + # Pod-specific + # spec.nodeName: ip-172-20-52-62.ec2.internal + - name: OPERATOR_POD_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # metadata.name: clickhouse-operator-6f87589dbb-ftcsf + - name: OPERATOR_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # status.podIP: 100.96.3.2 + - name: OPERATOR_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + # spec.serviceAccount: clickhouse-operator + # spec.serviceAccountName: clickhouse-operator + - name: OPERATOR_POD_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + # Container-specific + - name: OPERATOR_CONTAINER_CPU_REQUEST + valueFrom: + resourceFieldRef: + containerName: operator + resource: requests.cpu + - name: OPERATOR_CONTAINER_CPU_LIMIT + valueFrom: + resourceFieldRef: + containerName: operator + resource: limits.cpu + - name: OPERATOR_CONTAINER_MEM_REQUEST + valueFrom: + resourceFieldRef: + containerName: operator + resource: requests.memory + - name: OPERATOR_CONTAINER_MEM_LIMIT + valueFrom: + resourceFieldRef: + containerName: operator + resource: limits.memory + {{- range $key, $val := $.Spec.TelemetryStore.Spec.Env }} + - name: {{ $key }} + value: {{ $val }} + {{- end }} + ports: + - containerPort: 9999 + name: op-metrics + resources: + requests: + cpu: 1000m + memory: 2Gi + securityContext: + {} + - name: metrics-exporter + image: altinity/metrics-exporter:0.25.3 + imagePullPolicy: IfNotPresent + volumeMounts: + - name: etc-clickhouse-operator-folder + mountPath: /etc/clickhouse-operator + - name: etc-clickhouse-operator-confd-folder + mountPath: /etc/clickhouse-operator/chi/conf.d + - name: etc-clickhouse-operator-configd-folder + mountPath: /etc/clickhouse-operator/chi/config.d + - name: etc-clickhouse-operator-templatesd-folder + mountPath: /etc/clickhouse-operator/chi/templates.d + - name: etc-clickhouse-operator-usersd-folder + mountPath: /etc/clickhouse-operator/chi/users.d + - name: etc-keeper-operator-confd-folder + mountPath: /etc/clickhouse-operator/chk/conf.d + - name: etc-keeper-operator-configd-folder + mountPath: /etc/clickhouse-operator/chk/keeper_config.d + - name: etc-keeper-operator-templatesd-folder + mountPath: /etc/clickhouse-operator/chk/templates.d + - name: etc-keeper-operator-usersd-folder + mountPath: /etc/clickhouse-operator/chk/users.d + env: + # Pod-specific + # spec.nodeName: ip-172-20-52-62.ec2.internal + - name: OPERATOR_POD_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # metadata.name: clickhouse-operator-6f87589dbb-ftcsf + - name: OPERATOR_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # metadata.namespace: kube-system + - name: OPERATOR_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # status.podIP: 100.96.3.2 + - name: OPERATOR_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + # spec.serviceAccount: clickhouse-operator + # spec.serviceAccountName: clickhouse-operator + - name: OPERATOR_POD_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + # Container-specific + - name: OPERATOR_CONTAINER_CPU_REQUEST + valueFrom: + resourceFieldRef: + containerName: operator + resource: requests.cpu + - name: OPERATOR_CONTAINER_CPU_LIMIT + valueFrom: + resourceFieldRef: + containerName: operator + resource: limits.cpu + - name: OPERATOR_CONTAINER_MEM_REQUEST + valueFrom: + resourceFieldRef: + containerName: operator + resource: requests.memory + - name: OPERATOR_CONTAINER_MEM_LIMIT + valueFrom: + resourceFieldRef: + containerName: operator + resource: limits.memory + ports: + - containerPort: 8888 + name: ch-metrics + resources: + requests: + cpu: 100m + memory: 256Mi + securityContext: + {} diff --git a/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/kustomization.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/kustomization.yaml.gotmpl new file mode 100644 index 0000000..be4a5d4 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/kustomization.yaml.gotmpl @@ -0,0 +1,11 @@ +# based on manfiest https://github.com/Altinity/clickhouse-operator/releases/download/release-0.25.3/altinity-clickhouse-operator-0.25.3.tgz +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- serviceaccount.yaml +- configmap.yaml +- clusterrole.yaml +- clusterrolebinding.yaml +- service.yaml +- deployment.yaml diff --git a/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/service.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/service.yaml.gotmpl new file mode 100644 index 0000000..155cedf --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/service.yaml.gotmpl @@ -0,0 +1,18 @@ +{{- $name := print $.Metadata.Name "-telemetrystore-" $.Spec.TelemetryStore.Kind "-operator" -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }}-metrics + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $name }} + app.kubernetes.io/version: "0.25.3" +spec: + ports: + - port: 8888 + name: ch-metrics + - port: 9999 + name: op-metrics + selector: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $name }} diff --git a/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/serviceaccount.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/serviceaccount.yaml.gotmpl new file mode 100644 index 0000000..259960e --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/clickhouse-operator/serviceaccount.yaml.gotmpl @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + labels: + app.kubernetes.io/name: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + app.kubernetes.io/instance: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-operator + app.kubernetes.io/version: "0.25.3" diff --git a/internal/casting/kuberneteskustomizecasting/templates/ingester/configmap.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/ingester/configmap.yaml.gotmpl new file mode 100644 index 0000000..806ab8b --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/ingester/configmap.yaml.gotmpl @@ -0,0 +1,15 @@ +{{- $name := print $.Metadata.Name "-ingester" -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: ingester + app.kubernetes.io/managed-by: kustomize +data: +{{- range $filename, $content := $.Spec.Ingester.Spec.Config.Data }} + {{ $filename }}: |- +{{ $content | indent 4 }} +{{- end }} diff --git a/internal/casting/kuberneteskustomizecasting/templates/ingester/deployment.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/ingester/deployment.yaml.gotmpl new file mode 100644 index 0000000..da319e9 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/ingester/deployment.yaml.gotmpl @@ -0,0 +1,128 @@ +{{- $name := print $.Metadata.Name "-ingester" -}} +{{- $clickhouseName := print $.Metadata.Name "-clickhouse" -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: ingester + app.kubernetes.io/managed-by: kustomize +spec: + replicas: {{ int $.Spec.Ingester.Spec.Cluster.Replicas }} + selector: + matchLabels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: ingester + template: + metadata: + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: ingester + spec: + containers: + - args: + - "--config=/conf/ingester.yaml" + - "--manager-config=/conf/opamp.yaml" + - "--copy-path=/var/tmp/collector-config.yaml" + command: + - "/signoz-otel-collector" + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- range $key, $val := $.Spec.Ingester.Spec.Env }} + - name: {{ $key }} + value: {{ $val }} + {{- end }} + image: {{ $.Spec.Ingester.Spec.Image }} + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + port: 13133 + path: / + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + name: collector + ports: + - name: otlp-grpc + containerPort: 4317 + protocol: TCP + - name: otlp-http + containerPort: 4318 + protocol: TCP + - name: health + containerPort: 13133 + protocol: TCP + readinessProbe: + httpGet: + port: 13133 + path: / + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + resources: + requests: + cpu: 125m + memory: 512Mi + volumeMounts: + - name: ingester-config-vol + mountPath: /conf + initContainers: + - args: + - "migrate" + - "sync" + - "check" + command: + - "/signoz-otel-collector" + env: + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_CLUSTER + value: "cluster" + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_REPLICATION + value: "true" + {{- if and $.Spec.TelemetryStore.Spec.Enabled $.Spec.TelemetryStore.Status.Addresses.TCP (index $.Spec.TelemetryStore.Status.Addresses.TCP 0) }} + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN + value: {{ index $.Spec.TelemetryStore.Status.Addresses.TCP 0 }} + {{ end }} + {{- range $key, $val := $.Spec.Ingester.Spec.Env }} + - name: {{ $key }} + value: {{ $val }} + {{- end }} + image: {{ $.Spec.Ingester.Spec.Image }} + imagePullPolicy: IfNotPresent + name: check + serviceAccountName: {{ $name }} + volumes: + - configMap: + name: {{ $name }} + name: ingester-config-vol diff --git a/internal/casting/kuberneteskustomizecasting/templates/ingester/kustomization.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/ingester/kustomization.yaml.gotmpl new file mode 100644 index 0000000..9f50ec6 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/ingester/kustomization.yaml.gotmpl @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - serviceaccount.yaml + - configmap.yaml + - service.yaml + - deployment.yaml diff --git a/internal/casting/kuberneteskustomizecasting/templates/ingester/service.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/ingester/service.yaml.gotmpl new file mode 100644 index 0000000..b55fdaf --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/ingester/service.yaml.gotmpl @@ -0,0 +1,29 @@ +{{- $name := print $.Metadata.Name "-ingester" -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: ingester + app.kubernetes.io/managed-by: kustomize +spec: + type: ClusterIP + ports: + - name: otlp-grpc + port: 4317 + protocol: TCP + targetPort: 4317 + - name: otlp-http + port: 4318 + protocol: TCP + targetPort: 4318 + - name: health + port: 13133 + protocol: TCP + targetPort: 13133 + selector: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: ingester diff --git a/internal/casting/kuberneteskustomizecasting/templates/ingester/serviceaccount.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/ingester/serviceaccount.yaml.gotmpl new file mode 100644 index 0000000..cf81af4 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/ingester/serviceaccount.yaml.gotmpl @@ -0,0 +1,10 @@ +{{- $name := print $.Metadata.Name "-ingester" -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: ingester + app.kubernetes.io/managed-by: kustomize diff --git a/internal/casting/kuberneteskustomizecasting/templates/kustomization.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/kustomization.yaml.gotmpl new file mode 100644 index 0000000..a290fdb --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/kustomization.yaml.gotmpl @@ -0,0 +1,26 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: {{ $.Metadata.Name }} + +resources: +- namespace.yaml +{{- if $.Spec.TelemetryStore.Spec.Enabled }} +- clickhouse-operator +- telemetrystore/clickhouse +{{- end }} +{{- if $.Spec.TelemetryKeeper.Spec.Enabled }} +- telemetrykeeper/clickhousekeeper +{{- end }} +{{- if $.Spec.TelemetryStore.Spec.Enabled }} +- telemetrystore-migrator +{{- end }} +{{- if $.Spec.MetaStore.Spec.Enabled }} +- metastore/postgres +{{- end }} +{{- if $.Spec.Signoz.Spec.Enabled }} +- signoz +{{- end }} +{{- if $.Spec.Ingester.Spec.Enabled }} +- ingester +{{- end }} \ No newline at end of file diff --git a/internal/casting/kuberneteskustomizecasting/templates/metastore/postgres/kustomization.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/metastore/postgres/kustomization.yaml.gotmpl new file mode 100644 index 0000000..abca379 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/metastore/postgres/kustomization.yaml.gotmpl @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + +- serviceaccount.yaml +- service.yaml +- statefulset.yaml diff --git a/internal/casting/kuberneteskustomizecasting/templates/metastore/postgres/service.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/metastore/postgres/service.yaml.gotmpl new file mode 100644 index 0000000..54be7e0 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/metastore/postgres/service.yaml.gotmpl @@ -0,0 +1,21 @@ +{{- $name := print $.Metadata.Name "-metastore" -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/component: metastore + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: {{ $name }} +spec: + type: ClusterIP + ports: + - name: tcp-postgresql + port: 5432 + protocol: TCP + targetPort: tcp-postgresql + selector: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: metastore diff --git a/internal/casting/kuberneteskustomizecasting/templates/metastore/postgres/serviceaccount.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/metastore/postgres/serviceaccount.yaml.gotmpl new file mode 100644 index 0000000..5515a11 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/metastore/postgres/serviceaccount.yaml.gotmpl @@ -0,0 +1,10 @@ +{{- $name := print $.Metadata.Name "-metastore" -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/component: metastore + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: {{ $name }} diff --git a/internal/casting/kuberneteskustomizecasting/templates/metastore/postgres/statefulset.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/metastore/postgres/statefulset.yaml.gotmpl new file mode 100644 index 0000000..594fec7 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/metastore/postgres/statefulset.yaml.gotmpl @@ -0,0 +1,85 @@ +{{- $name := print $.Metadata.Name "-metastore" -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/component: metastore + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: {{ $name }} +spec: + serviceName: {{ $name }} + replicas: {{ int $.Spec.MetaStore.Spec.Cluster.Replicas }} + selector: + matchLabels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: metastore + template: + metadata: + labels: + app.kubernetes.io/component: metastore + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/name: {{ $name }} + spec: + containers: + - name: postgres + image: {{ $.Spec.MetaStore.Spec.Image }} + imagePullPolicy: IfNotPresent + ports: + - name: tcp-postgresql + containerPort: 5432 + protocol: TCP + env: + - name: POSTGRESQL_VOLUME_DIR + value: "/var/lib/postgresql" + - name: POSTGRESQL_PORT_NUMBER + value: "5432" + - name: PGDATA + value: "/var/lib/postgresql/data" + {{- range $key, $val := $.Spec.MetaStore.Spec.Env }} + - name: {{ $key }} + value: {{ $val }} + {{- end }} + livenessProbe: + exec: + command: + - /bin/sh + - -c + - exec pg_isready -U "$POSTGRES_USER" -h 127.0.0.1 -p 5432 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /bin/sh + - -c + - exec pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" -h 127.0.0.1 -p 5432 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 6 + resources: + requests: + cpu: 100m + memory: 128Mi + volumeMounts: + - name: pgdata + mountPath: /var/lib/postgresql + serviceAccountName: {{ $name }} + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: pgdata + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/internal/casting/kuberneteskustomizecasting/templates/namespace.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/namespace.yaml.gotmpl new file mode 100644 index 0000000..9b9cf69 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/namespace.yaml.gotmpl @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ $.Metadata.Name }} diff --git a/internal/casting/kuberneteskustomizecasting/templates/overrides/telemetrystore.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/overrides/telemetrystore.yaml.gotmpl new file mode 100644 index 0000000..2f96cf1 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/overrides/telemetrystore.yaml.gotmpl @@ -0,0 +1,70 @@ +async_load_databases: false +format_schema_path: /etc/clickhouse-server/config.d/ +user_defined_executable_functions_config: /etc/clickhouse-server/functions/custom-functions.xml +logger: + formatting: + type: json + names: + date_time: date_time + thread_name: thread_name + thread_id: thread_id + level: level + query_id: query_id + logger_name: logger_name + message: message + source_file: source_file + source_line: source_line +merge_tree: + storage_policy: data +profiles: + default: + allow_experimental_window_functions: "1" + allow_nondeterministic_mutations: "1" + max_threads: 16 + query_plan_max_limit_for_lazy_materialization: "0" + secondary_indices_enable_bulk_filtering: "0" + admin: + max_threads: 16 + query_plan_max_limit_for_lazy_materialization: "0" + secondary_indices_enable_bulk_filtering: "0" +prometheus: + endpoint: /metrics + port: 9363 +send_crash_reports: + enabled: false +storage_configuration: + disks: + default: {} + data0: + path: /mnt/data-0/ + keep_free_space_bytes: 10485760 + policies: + default: + volumes: + default: + disk: default + data: + volumes: + data0: + disk: data0 +text_log: + ttl: event_date + INTERVAL 3 DAY DELETE +latency_log: + ttl: event_date + INTERVAL 3 DAY DELETE +error_log: + ttl: event_date + INTERVAL 7 DAY DELETE +query_metric_log: + ttl: event_date + INTERVAL 3 DAY DELETE +users: + admin: + password: 27ff0399-0d3a-4bd8-919d-17c2181e6fb9 + networks: + ip: + - 10.0.0.0/8 + - 100.64.0.0/10 + - 172.16.0.0/12 + - 192.0.0.0/24 + - 198.18.0.0/15 + - 192.168.0.0/16 + profile: default + quota: default diff --git a/internal/casting/kuberneteskustomizecasting/templates/signoz/kustomization.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/signoz/kustomization.yaml.gotmpl new file mode 100644 index 0000000..abca379 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/signoz/kustomization.yaml.gotmpl @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + +- serviceaccount.yaml +- service.yaml +- statefulset.yaml diff --git a/internal/casting/kuberneteskustomizecasting/templates/signoz/service.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/signoz/service.yaml.gotmpl new file mode 100644 index 0000000..f5209b0 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/signoz/service.yaml.gotmpl @@ -0,0 +1,25 @@ +{{- $name := print $.Metadata.Name "-signoz" -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: signoz + app.kubernetes.io/managed-by: kustomize +spec: + type: ClusterIP + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: http + - name: opamp + port: 4320 + protocol: TCP + targetPort: opamp + selector: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: signoz diff --git a/internal/casting/kuberneteskustomizecasting/templates/signoz/serviceaccount.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/signoz/serviceaccount.yaml.gotmpl new file mode 100644 index 0000000..8cc9d2c --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/signoz/serviceaccount.yaml.gotmpl @@ -0,0 +1,10 @@ +{{- $name := print $.Metadata.Name "-signoz" -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: signoz + app.kubernetes.io/managed-by: kustomize diff --git a/internal/casting/kuberneteskustomizecasting/templates/signoz/statefulset.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/signoz/statefulset.yaml.gotmpl new file mode 100644 index 0000000..bbd8b8e --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/signoz/statefulset.yaml.gotmpl @@ -0,0 +1,65 @@ +{{- $name := print $.Metadata.Name "-signoz" -}} +{{- $clickhouseName := print $.Metadata.Name "-clickhouse" -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/component: signoz + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: {{ $name }} +spec: + replicas: {{ int $.Spec.Signoz.Spec.Cluster.Replicas }} + selector: + matchLabels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: signoz + serviceName: {{ $name }} + template: + metadata: + labels: + app.kubernetes.io/component: signoz + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/name: {{ $name }} + spec: + containers: + - env: + {{- range $key, $val := $.Spec.Signoz.Spec.Env }} + - name: {{ $key }} + value: {{ $val }} + {{- end }} + image: {{ $.Spec.Signoz.Spec.Image }} + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + port: http + path: /api/v1/health + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + name: signoz + ports: + - name: http + containerPort: 8080 + protocol: TCP + - name: opamp + containerPort: 4320 + protocol: TCP + readinessProbe: + httpGet: + port: http + path: /api/v1/health + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + resources: + requests: + cpu: 100m + memory: 200Mi + serviceAccountName: {{ $name }} \ No newline at end of file diff --git a/internal/casting/kuberneteskustomizecasting/templates/telemetrykeeper/clickhousekeeper/clickhousekeeperinstallation.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/telemetrykeeper/clickhousekeeper/clickhousekeeperinstallation.yaml.gotmpl new file mode 100644 index 0000000..f7fdc8f --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/telemetrykeeper/clickhousekeeper/clickhousekeeperinstallation.yaml.gotmpl @@ -0,0 +1,58 @@ +{{- $name := print $.Metadata.Name "-clickhouse-keeper" -}} +# Ref: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chk-examples/02-extended-1-node.yaml +apiVersion: clickhouse-keeper.altinity.com/v1 +kind: ClickHouseKeeperInstallation +metadata: + name: {{ $name }} +spec: + configuration: + clusters: + - name: "cluster" + layout: + replicasCount: {{ int $.Spec.TelemetryKeeper.Spec.Cluster.Replicas }} + settings: + logger/level: "trace" + logger/console: "true" + listen_host: "0.0.0.0" + keeper_server/four_letter_word_white_list: "*" + keeper_server/coordination_settings/raft_logs_level: "information" + defaults: + templates: + podTemplate: default + dataVolumeClaimTemplate: default + templates: + podTemplates: + - name: default + metadata: + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: telemetrykeeper + app.kubernetes.io/part-of: signoz + app.kubernetes.io/managed-by: kustomize + spec: + securityContext: + fsGroup: 101 + containers: + - name: clickhouse-keeper + imagePullPolicy: IfNotPresent + image: {{ $.Spec.TelemetryKeeper.Spec.Image }} + resources: + requests: + cpu: 1 + memory: 1Gi + {{- if $.Spec.TelemetryKeeper.Spec.Env}} + env: + {{- range $key, $val := $.Spec.TelemetryKeeper.Spec.Env }} + - name: {{ $key }} + value: {{ $val }} + {{- end }} + {{ end }} + volumeClaimTemplates: + - name: default + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/internal/casting/kuberneteskustomizecasting/templates/telemetrykeeper/clickhousekeeper/kustomization.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/telemetrykeeper/clickhousekeeper/kustomization.yaml.gotmpl new file mode 100644 index 0000000..e2d3083 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/telemetrykeeper/clickhousekeeper/kustomization.yaml.gotmpl @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- clickhousekeeperinstallation.yaml diff --git a/internal/casting/kuberneteskustomizecasting/templates/telemetrystore-migrator/job.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/telemetrystore-migrator/job.yaml.gotmpl new file mode 100644 index 0000000..117be80 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/telemetrystore-migrator/job.yaml.gotmpl @@ -0,0 +1,117 @@ +{{- $name := print $.Metadata.Name "-telemetrystore-migrator" -}} +{{- $clickhouseName := print $.Metadata.Name "-clickhouse" -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: schema-migrator + app.kubernetes.io/managed-by: kustomize +spec: + backoffLimit: 6 + template: + metadata: + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: schema-migrator + spec: + initContainers: + - name: ready + image: {{ $.Spec.Ingester.Spec.Image }} + imagePullPolicy: IfNotPresent + env: + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_CLUSTER + value: "cluster" + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_REPLICATION + value: "true" + {{- if and $.Spec.TelemetryStore.Spec.Enabled $.Spec.TelemetryStore.Status.Addresses.TCP (index $.Spec.TelemetryStore.Status.Addresses.TCP 0) }} + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN + value: {{ index $.Spec.TelemetryStore.Status.Addresses.TCP 0 }} + {{ end }} + {{- range $key, $val := $.Spec.Ingester.Spec.Env }} + - name: {{ $key }} + value: {{ $val }} + {{- end }} + args: + - "migrate" + - "ready" + resources: + requests: + cpu: 10m + memory: 16Mi + - name: bootstrap + image: {{ $.Spec.Ingester.Spec.Image }} + imagePullPolicy: IfNotPresent + env: + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_CLUSTER + value: "cluster" + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_REPLICATION + value: "true" + {{- if and $.Spec.TelemetryStore.Spec.Enabled $.Spec.TelemetryStore.Status.Addresses.TCP (index $.Spec.TelemetryStore.Status.Addresses.TCP 0) }} + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN + value: {{ index $.Spec.TelemetryStore.Status.Addresses.TCP 0 }} + {{ end }} + {{- range $key, $val := $.Spec.Ingester.Spec.Env }} + - name: {{ $key }} + value: {{ $val }} + {{- end }} + args: + - "migrate" + - "bootstrap" + resources: + requests: + cpu: 10m + memory: 16Mi + - name: sync + image: {{ $.Spec.Ingester.Spec.Image }} + imagePullPolicy: IfNotPresent + env: + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_CLUSTER + value: "cluster" + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_REPLICATION + value: "true" + {{- if and $.Spec.TelemetryStore.Spec.Enabled $.Spec.TelemetryStore.Status.Addresses.TCP (index $.Spec.TelemetryStore.Status.Addresses.TCP 0) }} + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN + value: {{ index $.Spec.TelemetryStore.Status.Addresses.TCP 0 }} + {{ end }} + {{- range $key, $val := $.Spec.Ingester.Spec.Env }} + - name: {{ $key }} + value: {{ $val }} + {{- end }} + args: + - "migrate" + - "sync" + - "up" + resources: + requests: + cpu: 10m + memory: 16Mi + containers: + - name: async + image: {{ $.Spec.Ingester.Spec.Image }} + imagePullPolicy: IfNotPresent + env: + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_CLUSTER + value: "cluster" + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_REPLICATION + value: "true" + {{- if and $.Spec.TelemetryStore.Spec.Enabled $.Spec.TelemetryStore.Status.Addresses.TCP (index $.Spec.TelemetryStore.Status.Addresses.TCP 0) }} + - name: SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN + value: {{ index $.Spec.TelemetryStore.Status.Addresses.TCP 0 }} + {{ end }} + {{- range $key, $val := $.Spec.Ingester.Spec.Env }} + - name: {{ $key }} + value: {{ $val }} + {{- end }} + args: + - "migrate" + - "async" + - "up" + resources: + requests: + cpu: 10m + memory: 16Mi + restartPolicy: OnFailure diff --git a/internal/casting/kuberneteskustomizecasting/templates/telemetrystore-migrator/kustomization.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/telemetrystore-migrator/kustomization.yaml.gotmpl new file mode 100644 index 0000000..2868062 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/telemetrystore-migrator/kustomization.yaml.gotmpl @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - job.yaml diff --git a/internal/casting/kuberneteskustomizecasting/templates/telemetrystore/clickhouse/clickhouseinstallation.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/telemetrystore/clickhouse/clickhouseinstallation.yaml.gotmpl new file mode 100644 index 0000000..c4ae583 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/telemetrystore/clickhouse/clickhouseinstallation.yaml.gotmpl @@ -0,0 +1,213 @@ +{{- $name := print $.Metadata.Name "-clickhouse" -}} +{{- $keeperName := print $.Metadata.Name "-clickhouse-keeper" -}} +apiVersion: clickhouse.altinity.com/v1 +kind: ClickHouseInstallation +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/version: {{ $.Spec.TelemetryStore.Spec.Version | quote }} + app.kubernetes.io/component: telemetrystore + app.kubernetes.io/part-of: signoz + app.kubernetes.io/managed-by: kustomize +spec: + configuration: + files: +{{- $configData := $.Spec.TelemetryStore.Status.Config.Data }} +{{- $config := index $configData "config-0-0.yaml" | fromYaml }} +{{- if $config }} +{{- $files := omit $config "profiles" "users" "quotas" "remote_servers" "zookeeper" "macros" "tcp_port" "http_port" "interserver_http_port" "listen_host" "display_name" "distributed_ddl" }} + config.d/crash.yaml: | +{{ pick $files "send_crash_reports" | toYaml | indent 8 }} +{{- $files = omit $files "send_crash_reports" }} + config.d/formatting.yaml: | +{{ pick $files "logger" | toYaml | indent 8 }} +{{- $files = omit $files "logger" }} + config.d/load.yaml: | +{{ pick $files "async_load_databases" | toYaml | indent 8 }} +{{- $files = omit $files "async_load_databases" }} + config.d/path.yaml: | +{{ pick $files "path" "merge_tree" | toYaml | indent 8 }} +{{- $files = omit $files "path" "merge_tree" }} + config.d/storage.yaml: | +{{ pick $files "storage_configuration" | toYaml | indent 8 }} +{{- $files = omit $files "storage_configuration" }} + config.d/system_log.yaml: | +{{ pick $files "text_log" "latency_log" "error_log" "query_metric_log" | toYaml | indent 8 }} +{{- $files = omit $files "text_log" "latency_log" "error_log" "query_metric_log" }} + config.d/extra.yaml: | +{{ $files | toYaml | indent 8 }} +{{- end }} + events.proto: | + syntax = "proto3"; + message Event { + string uuid = 1; + string event = 2; + string properties = 3; + string timestamp = 4; + uint64 team_id = 5; + string distinct_id = 6; + string created_at = 7; + string elements_chain = 8; + } +{{- if $config }} + profiles: +{{ index $config "profiles" | flattenKeys | toYaml | indent 6 }} + users: +{{ index $config "users" | flattenKeys | toYaml | indent 6 }} + quotas: +{{ index $config "quotas" | flattenKeys | toYaml | indent 6 }} +{{- end }} + clusters: + - name: cluster + layout: + replicasCount: {{ int $.Spec.TelemetryStore.Spec.Cluster.Replicas }} + shardsCount: {{ int $.Spec.TelemetryStore.Spec.Cluster.Shards }} + zookeeper: + nodes: + - host: keeper-{{ $keeperName }} + port: 2181 + defaults: + templates: + podTemplate: default + serviceTemplate: default + templates: + podTemplates: + - name: default + metadata: + labels: + app.kubernetes.io/component: telemetrystore + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/version: {{ $.Spec.TelemetryStore.Spec.Version | quote }} + spec: + containers: + - command: + - /bin/bash + - -c + - /usr/bin/clickhouse-server --config-file=/etc/clickhouse-server/config.xml + image: {{ $.Spec.TelemetryStore.Spec.Image }} + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 10 + httpGet: + path: /ping + port: http + initialDelaySeconds: 60 + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 1 + name: clickhouse + ports: + - containerPort: 8123 + name: http + - containerPort: 9000 + name: client + - containerPort: 9009 + name: interserver + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ping + port: http + initialDelaySeconds: 10 + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 500m + memory: 512Mi + startupProbe: + failureThreshold: 30 + httpGet: + path: /ping + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /var/lib/clickhouse/user_scripts + name: shared-binary-volume + - mountPath: /etc/clickhouse-server/functions + name: custom-functions-volume + - mountPath: /var/lib/clickhouse + name: default + - mountPath: /mnt/data-0 + name: data-0 + {{- if $.Spec.TelemetryStore.Spec.Env }} + env: + {{- range $key, $val := $.Spec.TelemetryStore.Spec.Env }} + - name: {{ $key }} + value: {{ $val }} + {{- end }} + {{ end }} + initContainers: + - name: histogram + command: + - sh + - -c + - | + set -e + version="v0.0.1" + node_os=$(uname -s | tr '[:upper:]' '[:lower:]') + node_arch=$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) + echo "Fetching histogram-binary for ${node_os}/${node_arch}" + cd /tmp + wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F${version}/histogram-quantile_${node_os}_${node_arch}.tar.gz" + tar -xzf histogram-quantile.tar.gz + chmod +x histogram-quantile + mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile + echo "histogram-quantile installed successfully" + image: docker.io/alpine:3.18.2 + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /var/lib/clickhouse/user_scripts + name: shared-binary-volume + securityContext: + fsGroup: 101 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 101 + runAsUser: 101 + volumes: + - emptyDir: {} + name: shared-binary-volume + - configMap: + name: {{ $name }}-custom-functions + name: custom-functions-volume + serviceTemplates: + - name: default + generateName: {{ $name }} + metadata: + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: telemetrystore + app.kubernetes.io/part-of: signoz + app.kubernetes.io/managed-by: kustomize + spec: + ports: + - name: http + port: 8123 + - name: tcp + port: 9000 + type: ClusterIP + volumeClaimTemplates: + - name: data-0 + reclaimPolicy: Retain + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 25Gi + - name: default + reclaimPolicy: Retain + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/internal/casting/kuberneteskustomizecasting/templates/telemetrystore/clickhouse/configmap.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/telemetrystore/clickhouse/configmap.yaml.gotmpl new file mode 100644 index 0000000..1ec8f6a --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/telemetrystore/clickhouse/configmap.yaml.gotmpl @@ -0,0 +1,14 @@ +{{- $name := print $.Metadata.Name "-clickhouse" -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-custom-functions + labels: + app.kubernetes.io/name: {{ $name }} + app.kubernetes.io/instance: {{ $.Metadata.Name }} + app.kubernetes.io/component: telemetrystore + app.kubernetes.io/part-of: signoz + app.kubernetes.io/managed-by: kustomize +data: + custom-functions.xml: | +{{ index $.Spec.TelemetryStore.Spec.Config.Data "functions.yaml" | indent 4 }} \ No newline at end of file diff --git a/internal/casting/kuberneteskustomizecasting/templates/telemetrystore/clickhouse/kustomization.yaml.gotmpl b/internal/casting/kuberneteskustomizecasting/templates/telemetrystore/clickhouse/kustomization.yaml.gotmpl new file mode 100644 index 0000000..a6835a5 --- /dev/null +++ b/internal/casting/kuberneteskustomizecasting/templates/telemetrystore/clickhouse/kustomization.yaml.gotmpl @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- configmap.yaml +- clickhouseinstallation.yaml diff --git a/internal/casting/railwaytemplatecasting/casting.go b/internal/casting/railwaytemplatecasting/casting.go new file mode 100644 index 0000000..f16eb49 --- /dev/null +++ b/internal/casting/railwaytemplatecasting/casting.go @@ -0,0 +1,172 @@ +package railwaytemplatecasting + +import ( + "bytes" + "context" + "log/slog" + "path/filepath" + + "github.com/signoz/foundry/api/v1alpha1/installation" + "github.com/signoz/foundry/internal/casting" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/molding" +) + +var _ casting.Casting = (*railwayTemplateCasting)(nil) + +type railwayTemplateCasting struct { + logger *slog.Logger + castings []*domain.Template +} + +func New(logger *slog.Logger) *railwayTemplateCasting { + return &railwayTemplateCasting{ + logger: logger, + castings: []*domain.Template{ + telemetryKeeperDockerfileTemplate, + telemetryStoreDockerfileTemplate, + ingesterDockerfileTemplate, + signozDockerfileTemplate, + telemetryStoreMigratorDockerfileTemplate, + railwayTelemetryKeeperTemplate, + railwayTelemetryStoreTemplate, + railwayIngesterTemplate, + railwaySignozTemplate, + railwayTelemetryStoreMigratorTemplate, + telemetryKeeperOverrideTemplate, + telemetryStoreOverrideTemplate, + }, + } +} + +func (c *railwayTemplateCasting) Enricher(ctx context.Context, config *installation.Casting) (molding.MoldingEnricher, error) { + return newRailwayTemplateMoldingEnricher(config) +} + +func (c *railwayTemplateCasting) Forge(ctx context.Context, config installation.Casting, poursPath string) ([]domain.Material, error) { + var materials []domain.Material + + // TelemetryKeeper: Dockerfile + configs + railway.json + if config.Spec.TelemetryKeeper.Spec.IsEnabled() { + dockerfileBuf := bytes.NewBuffer(nil) + if err := telemetryKeeperDockerfileTemplate.Execute(dockerfileBuf, config); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "telemetrykeeper dockerfile") + } + materials = append(materials, domain.NewBlobMaterial(dockerfileBuf.Bytes(), filepath.Join(casting.DeploymentDir, "telemetrykeeper/Dockerfile"))) + railwayBuf := bytes.NewBuffer(nil) + if err := railwayTelemetryKeeperTemplate.Execute(railwayBuf, config); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "telemetrykeeper railway.json") + } + materials = append(materials, domain.NewBlobMaterial(railwayBuf.Bytes(), filepath.Join(casting.DeploymentDir, "telemetrykeeper/railway.json"))) + for filename, content := range config.Spec.TelemetryKeeper.Spec.Config.Data { + m, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(casting.DeploymentDir, "telemetrykeeper/keeper.d/", filename)) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "telemetrykeeper config") + } + materials = append(materials, m) + } + } + + // TelemetryStore: Dockerfile + configs + railway.json + if config.Spec.TelemetryStore.Spec.IsEnabled() { + dockerfileBuf := bytes.NewBuffer(nil) + if err := telemetryStoreDockerfileTemplate.Execute(dockerfileBuf, config); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "telemetrystore dockerfile") + } + materials = append(materials, domain.NewBlobMaterial(dockerfileBuf.Bytes(), filepath.Join(casting.DeploymentDir, "telemetrystore/Dockerfile"))) + railwayBuf := bytes.NewBuffer(nil) + if err := railwayTelemetryStoreTemplate.Execute(railwayBuf, config); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "telemetrystore railway.json") + } + materials = append(materials, domain.NewBlobMaterial(railwayBuf.Bytes(), filepath.Join(casting.DeploymentDir, "telemetrystore/railway.json"))) + for filename, content := range config.Spec.TelemetryStore.Spec.Config.Data { + m, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(casting.DeploymentDir, "telemetrystore/config.d/", filename)) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "telemetrystore config") + } + materials = append(materials, m) + } + } + + // Ingester: Dockerfile + configs + railway.json + if config.Spec.Ingester.Spec.IsEnabled() { + dockerfileBuf := bytes.NewBuffer(nil) + if err := ingesterDockerfileTemplate.Execute(dockerfileBuf, config); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "ingester dockerfile") + } + materials = append(materials, domain.NewBlobMaterial(dockerfileBuf.Bytes(), filepath.Join(casting.DeploymentDir, "ingester/Dockerfile"))) + railwayBuf := bytes.NewBuffer(nil) + if err := railwayIngesterTemplate.Execute(railwayBuf, config); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "ingester railway.json") + } + materials = append(materials, domain.NewBlobMaterial(railwayBuf.Bytes(), filepath.Join(casting.DeploymentDir, "ingester/railway.json"))) + for filename, content := range config.Spec.Ingester.Spec.Config.Data { + m, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(casting.DeploymentDir, "ingester/", filename)) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "ingester config") + } + materials = append(materials, m) + } + } + + // Signoz: Dockerfile + railway.json + if config.Spec.Signoz.Spec.IsEnabled() { + dockerfileBuf := bytes.NewBuffer(nil) + if err := signozDockerfileTemplate.Execute(dockerfileBuf, config); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "signoz dockerfile") + } + materials = append(materials, domain.NewBlobMaterial(dockerfileBuf.Bytes(), filepath.Join(casting.DeploymentDir, "signoz/Dockerfile"))) + railwayBuf := bytes.NewBuffer(nil) + if err := railwaySignozTemplate.Execute(railwayBuf, config); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "signoz railway.json") + } + materials = append(materials, domain.NewBlobMaterial(railwayBuf.Bytes(), filepath.Join(casting.DeploymentDir, "signoz/railway.json"))) + } + + // TelemetryStore migrator: Dockerfile + railway.json + if config.Spec.TelemetryStore.Spec.IsEnabled() { + dockerfileBuf := bytes.NewBuffer(nil) + if err := telemetryStoreMigratorDockerfileTemplate.Execute(dockerfileBuf, config); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "telemetrystore-migrator dockerfile") + } + materials = append(materials, domain.NewBlobMaterial(dockerfileBuf.Bytes(), filepath.Join(casting.DeploymentDir, "telemetrystore-migrator/Dockerfile"))) + railwayBuf := bytes.NewBuffer(nil) + if err := railwayTelemetryStoreMigratorTemplate.Execute(railwayBuf, config); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "telemetrystore-migrator railway.json") + } + materials = append(materials, domain.NewBlobMaterial(railwayBuf.Bytes(), filepath.Join(casting.DeploymentDir, "telemetrystore-migrator/railway.json"))) + } + + return materials, nil +} + +func (c *railwayTemplateCasting) Cast(ctx context.Context, config installation.Casting, poursPath string) error { + c.logger.InfoContext(ctx, "Please use the template.") + return nil +} + +func getRailwayMaterial(config *installation.Casting) ([]domain.StructuredMaterial, error) { + var materials []domain.StructuredMaterial + + keeperBuf := bytes.NewBuffer(nil) + if err := telemetryKeeperOverrideTemplate.Execute(keeperBuf, config); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to execute keeper override template") + } + keeperMaterial, err := domain.NewYAMLMaterial(keeperBuf.Bytes(), "keeper_overrides.yaml") + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to create keeper override material") + } + materials = append(materials, keeperMaterial) + + storeBuf := bytes.NewBuffer(nil) + if err := telemetryStoreOverrideTemplate.Execute(storeBuf, config); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to execute keeper override template") + } + storeMaterial, err := domain.NewYAMLMaterial(storeBuf.Bytes(), "store_overrides.yaml") + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to create keeper override material") + } + materials = append(materials, storeMaterial) + return materials, nil +} diff --git a/internal/casting/railwaytemplatecasting/embed.go b/internal/casting/railwaytemplatecasting/embed.go new file mode 100644 index 0000000..190e3d7 --- /dev/null +++ b/internal/casting/railwaytemplatecasting/embed.go @@ -0,0 +1,28 @@ +package railwaytemplatecasting + +import ( + "embed" + + "github.com/signoz/foundry/internal/domain" +) + +//go:embed templates/*.gotmpl +var templates embed.FS + +var ( + telemetryKeeperDockerfileTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/Dockerfile.clickhousekeeper.telemetrykeeper.v2556.gotmpl", domain.FormatText) + telemetryStoreDockerfileTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/Dockerfile.clickhouse.telemetrystore.v2556.gotmpl", domain.FormatText) + ingesterDockerfileTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/Dockerfile.ingester.gotmpl", domain.FormatText) + signozDockerfileTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/Dockerfile.signoz.gotmpl", domain.FormatText) + telemetryStoreMigratorDockerfileTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/Dockerfile.telemetrystore-migrator.gotmpl", domain.FormatText) + + railwayTelemetryKeeperTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/railway.telemetrykeeper.json.gotmpl", domain.FormatText) + railwayTelemetryStoreTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/railway.telemetrystore.json.gotmpl", domain.FormatText) + railwayIngesterTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/railway.ingester.json.gotmpl", domain.FormatText) + railwaySignozTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/railway.signoz.json.gotmpl", domain.FormatText) + railwayTelemetryStoreMigratorTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/railway.telemetrystore-migrator.json.gotmpl", domain.FormatText) + + // molding overrides. + telemetryKeeperOverrideTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/telemetrykeeper.yaml.gotmpl", domain.FormatYAML) + telemetryStoreOverrideTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/telemetrystore.yaml.gotmpl", domain.FormatYAML) +) diff --git a/internal/casting/railwaytemplatecasting/enricher.go b/internal/casting/railwaytemplatecasting/enricher.go new file mode 100644 index 0000000..aff3c49 --- /dev/null +++ b/internal/casting/railwaytemplatecasting/enricher.go @@ -0,0 +1,80 @@ +package railwaytemplatecasting + +import ( + "context" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/installation" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/molding" +) + +var _ molding.MoldingEnricher = (*railwayTemplateMoldingEnricher)(nil) + +type railwayTemplateMoldingEnricher struct { + material []domain.StructuredMaterial +} + +func newRailwayTemplateMoldingEnricher(config *installation.Casting) (*railwayTemplateMoldingEnricher, error) { + material, err := getRailwayMaterial(config) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to get compose yaml material") + } + return &railwayTemplateMoldingEnricher{material: material}, nil +} + +// railwayInternalHost returns the Railway private DNS hostname for a service. +// Railway services communicate via SERVICE_NAME.railway.internal within the same project. +func railwayInternalHost(serviceName string) string { + return serviceName + ".railway.internal" +} + +func (enricher *railwayTemplateMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *installation.Casting) error { + name := config.Metadata.Name + if name == "" { + name = "signoz" + } + switch kind { + case v1alpha1.MoldingKindTelemetryStore: + if !config.Spec.TelemetryStore.Spec.IsEnabled() { + return nil + } + svc := name + "-telemetrystore-" + config.Spec.TelemetryStore.Kind.String() + config.Spec.TelemetryStore.Status.Addresses.TCP = []string{domain.MustNewAddress("tcp", railwayInternalHost(svc), 9000).String()} + if config.Spec.TelemetryStore.Status.Extras == nil { + config.Spec.TelemetryStore.Status.Extras = make(map[string]string) + } + config.Spec.TelemetryStore.Status.Extras["service_names"] = svc + config.Spec.TelemetryStore.Status.Extras["_overrides"] = string(enricher.material[1].FmtContents()) + + case v1alpha1.MoldingKindSignoz: + if !config.Spec.Signoz.Spec.IsEnabled() { + return nil + } + svc := name + "-signoz" + config.Spec.Signoz.Status.Addresses.APIServer = []string{domain.MustNewAddress("tcp", railwayInternalHost(svc), 8080).String()} + config.Spec.Signoz.Status.Addresses.Opamp = []string{domain.MustNewAddress("ws", railwayInternalHost(svc), 4320).String()} + + case v1alpha1.MoldingKindTelemetryKeeper: + if !config.Spec.TelemetryKeeper.Spec.IsEnabled() { + return nil + } + svc := name + "-telemetrykeeper-" + config.Spec.TelemetryKeeper.Kind.String() + config.Spec.TelemetryKeeper.Status.Addresses.Client = []string{domain.MustNewAddress("tcp", railwayInternalHost(svc), 9181).String()} + config.Spec.TelemetryKeeper.Status.Addresses.Raft = []string{domain.MustNewAddress("tcp", railwayInternalHost(svc), 9234).String()} + if config.Spec.TelemetryKeeper.Status.Extras == nil { + config.Spec.TelemetryKeeper.Status.Extras = make(map[string]string) + } + config.Spec.TelemetryKeeper.Status.Extras["service_names"] = svc + config.Spec.TelemetryKeeper.Status.Extras["_overrides"] = string(enricher.material[0].FmtContents()) + case v1alpha1.MoldingKindIngester: + if !config.Spec.Ingester.Spec.IsEnabled() { + return nil + } + svc := name + "-ingester" + config.Spec.Ingester.Status.Addresses.OTLP = []string{domain.MustNewAddress("tcp", railwayInternalHost(svc), 4318).String()} + } + + return nil +} diff --git a/internal/casting/railwaytemplatecasting/templates/Dockerfile.clickhouse.telemetrystore.v2556.gotmpl b/internal/casting/railwaytemplatecasting/templates/Dockerfile.clickhouse.telemetrystore.v2556.gotmpl new file mode 100644 index 0000000..705500b --- /dev/null +++ b/internal/casting/railwaytemplatecasting/templates/Dockerfile.clickhouse.telemetrystore.v2556.gotmpl @@ -0,0 +1,40 @@ +FROM {{$.Spec.TelemetryStore.Spec.Image }} + +COPY config.d/ /etc/clickhouse-server/ + +RUN mkdir -p /etc/clickhouse-server/user_scripts/ \ + && chown clickhouse:clickhouse /etc/clickhouse-server/user_scripts + +ARG VERSION=v0.0.1 + +ENV CLICKHOUSE_CONFIG=/etc/clickhouse-server/config-0-0.yaml +ENV CLICKHOUSE_SKIP_USER_SETUP=1 + +{{- range $key, $value := $.Spec.TelemetryStore.Spec.Env }} +ENV {{ $key }}={{ $value }} +{{- end }} + +# Download and install histogram-quantile binary +RUN cat > /tmp/install-histogram-quantile.sh <<'EOF' +#!/bin/bash +set -euo pipefail + +node_os=$(uname -s | tr '[:upper:]' '[:lower:]') +node_arch=$(uname -m | sed 's/aarch64/arm64/' | sed 's/x86_64/amd64/') + +echo "Fetching histogram-quantile for ${node_os}/${node_arch}" + +cd /tmp +wget -O histogram-quantile.tar.gz \ + "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F${VERSION}/histogram-quantile_${node_os}_${node_arch}.tar.gz" + +tar -xzf histogram-quantile.tar.gz +mv histogram-quantile /etc/clickhouse-server/user_scripts/histogramQuantile +rm histogram-quantile.tar.gz + +echo "histogram-quantile installed successfully" +EOF + +RUN chmod +x /tmp/install-histogram-quantile.sh \ + && /tmp/install-histogram-quantile.sh \ + && rm /tmp/install-histogram-quantile.sh \ No newline at end of file diff --git a/internal/casting/railwaytemplatecasting/templates/Dockerfile.clickhousekeeper.telemetrykeeper.v2556.gotmpl b/internal/casting/railwaytemplatecasting/templates/Dockerfile.clickhousekeeper.telemetrykeeper.v2556.gotmpl new file mode 100644 index 0000000..0802955 --- /dev/null +++ b/internal/casting/railwaytemplatecasting/templates/Dockerfile.clickhousekeeper.telemetrykeeper.v2556.gotmpl @@ -0,0 +1,30 @@ +FROM {{ $.Spec.TelemetryKeeper.Spec.Image }} + +# Copy all keeper config files +COPY keeper.d/ /etc/clickhouse-keeper/ + +{{- range $key, $value := $.Spec.TelemetryKeeper.Spec.Env }} +ENV {{ $key }}={{ $value }} +{{- end }} +# Create entrypoint script to dynamically select config based on replica index +RUN cat > /entrypoint.sh <<'EOF' +#!/bin/bash +set -euo pipefail + +# Extract replica index from service name, default to 0 +REPLICA_INDEX=$(echo "${RAILWAY_SERVICE_NAME:-}" | grep -oE "[0-9]+$" || echo "0") +CONFIG_FILE="/etc/clickhouse-keeper/keeper-${REPLICA_INDEX}.yaml" + +# Fallback to keeper-0.yaml if replica-specific config doesn't exist +if [ ! -f "$CONFIG_FILE" ]; then + echo "Config $CONFIG_FILE not found, falling back to keeper-0.yaml" + CONFIG_FILE="/etc/clickhouse-keeper/keeper-0.yaml" +fi + +echo "Starting ClickHouse Keeper with config: $CONFIG_FILE" +exec /usr/bin/clickhouse-keeper --config-file="$CONFIG_FILE" "$@" +EOF + +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/internal/casting/railwaytemplatecasting/templates/Dockerfile.ingester.gotmpl b/internal/casting/railwaytemplatecasting/templates/Dockerfile.ingester.gotmpl new file mode 100644 index 0000000..b1291a5 --- /dev/null +++ b/internal/casting/railwaytemplatecasting/templates/Dockerfile.ingester.gotmpl @@ -0,0 +1,11 @@ +FROM {{ $.Spec.Ingester.Spec.Image }} + +COPY ingester.yaml /etc/otel-collector-config.yaml +COPY opamp.yaml /etc/opamp-config.yaml + +{{ if and $.Spec.TelemetryStore.Spec.Enabled $.Spec.TelemetryStore.Status.Addresses.TCP (index $.Spec.TelemetryStore.Status.Addresses.TCP 0) }} +ENV SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN={{ index $.Spec.TelemetryStore.Status.Addresses.TCP 0 }} +{{ end }} +{{- range $key, $value := $.Spec.Ingester.Spec.Env }} +ENV {{ $key }}={{ $value }} +{{- end }} \ No newline at end of file diff --git a/internal/casting/railwaytemplatecasting/templates/Dockerfile.signoz.gotmpl b/internal/casting/railwaytemplatecasting/templates/Dockerfile.signoz.gotmpl new file mode 100644 index 0000000..59f6a09 --- /dev/null +++ b/internal/casting/railwaytemplatecasting/templates/Dockerfile.signoz.gotmpl @@ -0,0 +1,6 @@ +FROM {{ $.Spec.Signoz.Spec.Image }} + + +{{- range $key, $value := $.Spec.Signoz.Spec.Env }} +ENV {{ $key }}={{ $value }} +{{- end }} \ No newline at end of file diff --git a/internal/casting/railwaytemplatecasting/templates/Dockerfile.telemetrystore-migrator.gotmpl b/internal/casting/railwaytemplatecasting/templates/Dockerfile.telemetrystore-migrator.gotmpl new file mode 100644 index 0000000..ec0ab6a --- /dev/null +++ b/internal/casting/railwaytemplatecasting/templates/Dockerfile.telemetrystore-migrator.gotmpl @@ -0,0 +1,9 @@ +FROM {{ $.Spec.Ingester.Spec.Image }} + +{{ if and $.Spec.TelemetryStore.Spec.Enabled $.Spec.TelemetryStore.Status.Addresses.TCP (index $.Spec.TelemetryStore.Status.Addresses.TCP 0) }} +ENV SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN={{ index $.Spec.TelemetryStore.Status.Addresses.TCP 0 }} +{{ end }} + +{{- range $key, $value := $.Spec.Ingester.Spec.Env }} +ENV {{ $key }}={{ $value }} +{{- end }} \ No newline at end of file diff --git a/internal/casting/railwaytemplatecasting/templates/railway.ingester.json.gotmpl b/internal/casting/railwaytemplatecasting/templates/railway.ingester.json.gotmpl new file mode 100644 index 0000000..0e3bdf5 --- /dev/null +++ b/internal/casting/railwaytemplatecasting/templates/railway.ingester.json.gotmpl @@ -0,0 +1,12 @@ +{ + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "Dockerfile" + }, + "deploy": { + "numReplicas": {{ max 1 (default 1 $.Spec.Ingester.Spec.Cluster.Replicas) }}, + "startCommand": "/bin/sh -c '/signoz-otel-collector migrate sync check && /signoz-otel-collector --config=/etc/otel-collector-config.yaml --manager-config=/etc/opamp-config.yaml --copy-path=/var/tmp/collector-config.yaml'", + "restartPolicyType": "ON_FAILURE", + "restartPolicyMaxRetries": 5 + } +} diff --git a/internal/casting/railwaytemplatecasting/templates/railway.signoz.json.gotmpl b/internal/casting/railwaytemplatecasting/templates/railway.signoz.json.gotmpl new file mode 100644 index 0000000..32c190f --- /dev/null +++ b/internal/casting/railwaytemplatecasting/templates/railway.signoz.json.gotmpl @@ -0,0 +1,13 @@ +{ + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "Dockerfile" + }, + "deploy": { + "numReplicas": {{ default 1 $.Spec.Signoz.Spec.Cluster.Replicas }}, + "healthcheckPath": "/api/v1/health", + "healthcheckTimeout": 300, + "restartPolicyType": "ON_FAILURE", + "restartPolicyMaxRetries": 5 + } +} diff --git a/internal/casting/railwaytemplatecasting/templates/railway.telemetrykeeper.json.gotmpl b/internal/casting/railwaytemplatecasting/templates/railway.telemetrykeeper.json.gotmpl new file mode 100644 index 0000000..84435fb --- /dev/null +++ b/internal/casting/railwaytemplatecasting/templates/railway.telemetrykeeper.json.gotmpl @@ -0,0 +1,14 @@ +{ + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "Dockerfile" + }, + "deploy": { + "numReplicas": 1, + "healthcheckPath": "/ready", + "healthcheckTimeout": 300, + "restartPolicyType": "ON_FAILURE", + "restartPolicyMaxRetries": 5, + "requiredMountPath": "/var/lib/clickhouse-keeper/" + } +} diff --git a/internal/casting/railwaytemplatecasting/templates/railway.telemetrystore-migrator.json.gotmpl b/internal/casting/railwaytemplatecasting/templates/railway.telemetrystore-migrator.json.gotmpl new file mode 100644 index 0000000..840ad30 --- /dev/null +++ b/internal/casting/railwaytemplatecasting/templates/railway.telemetrystore-migrator.json.gotmpl @@ -0,0 +1,11 @@ +{ + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "Dockerfile" + }, + "deploy": { + "numReplicas": 1, + "startCommand": "/bin/sh -c '/signoz-otel-collector migrate ready && /signoz-otel-collector migrate bootstrap && /signoz-otel-collector migrate sync up && /signoz-otel-collector migrate async up'", + "restartPolicyType": "NEVER" + } +} diff --git a/internal/casting/railwaytemplatecasting/templates/railway.telemetrystore.json.gotmpl b/internal/casting/railwaytemplatecasting/templates/railway.telemetrystore.json.gotmpl new file mode 100644 index 0000000..2137a49 --- /dev/null +++ b/internal/casting/railwaytemplatecasting/templates/railway.telemetrystore.json.gotmpl @@ -0,0 +1,14 @@ +{ + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "Dockerfile" + }, + "deploy": { + "numReplicas": 1, + "healthcheckPath": "/ping", + "healthcheckTimeout": 300, + "restartPolicyType": "ON_FAILURE", + "restartPolicyMaxRetries": 5, + "requiredMountPath": "/var/lib/clickhouse/" + } +} diff --git a/internal/casting/railwaytemplatecasting/templates/telemetrykeeper.yaml.gotmpl b/internal/casting/railwaytemplatecasting/templates/telemetrykeeper.yaml.gotmpl new file mode 100644 index 0000000..d20bb18 --- /dev/null +++ b/internal/casting/railwaytemplatecasting/templates/telemetrykeeper.yaml.gotmpl @@ -0,0 +1,10 @@ +listen_host: "::" +keeper_server: + raft_configuration: + server: + - hostname: localhost + port: 9234 + id: 0 + http_control: + port: 9182 + endpoint: /ready \ No newline at end of file diff --git a/internal/casting/railwaytemplatecasting/templates/telemetrystore.yaml.gotmpl b/internal/casting/railwaytemplatecasting/templates/telemetrystore.yaml.gotmpl new file mode 100644 index 0000000..a9385bd --- /dev/null +++ b/internal/casting/railwaytemplatecasting/templates/telemetrystore.yaml.gotmpl @@ -0,0 +1,8 @@ +listen_host: "::" +remote_servers: + cluster: + shard: + - replica: + host: localhost + port: 9000 +user_scripts_path: /etc/clickhouse-server/user_scripts/ \ No newline at end of file diff --git a/internal/casting/rendercasting/casting.go b/internal/casting/rendercasting/casting.go index 8fbc8c9..e22b6c4 100644 --- a/internal/casting/rendercasting/casting.go +++ b/internal/casting/rendercasting/casting.go @@ -17,13 +17,13 @@ var _ casting.Casting = (*renderCasting)(nil) type renderCasting struct { logger *slog.Logger - castings []*types.Template + castings []*domain.Template } func New(logger *slog.Logger) *renderCasting { return &renderCasting{ logger: logger, - castings: []*types.Template{ + castings: []*domain.Template{ renderYAMLTemplate, telemetryKeeperDockerfileTemplate, telemetryStoreDockerfileTemplate, @@ -32,75 +32,75 @@ func New(logger *slog.Logger) *renderCasting { } } -func (c *renderCasting) Enricher(ctx context.Context, config *v1alpha1.Casting) (molding.MoldingEnricher, error) { +func (c *renderCasting) Enricher(ctx context.Context, config *installation.Casting) (molding.MoldingEnricher, error) { return newRenderMoldingEnricher(config) } -func (c *renderCasting) Forge(ctx context.Context, config v1alpha1.Casting, poursPath string) ([]types.Material, error) { - var materials []types.Material +func (c *renderCasting) Forge(ctx context.Context, config installation.Casting, poursPath string) ([]domain.Material, error) { + var materials []domain.Material configsDir := filepath.Join(casting.DeploymentDir, "configs/") // Generate render.yaml blueprintMaterial, err := getRenderMaterial(&config, filepath.Join(casting.DeploymentDir, "render.yaml")) if err != nil { - return nil, fmt.Errorf("failed to create blueprint yaml material: %w", err) + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to create blueprint yaml material") } materials = append(materials, blueprintMaterial) // Generate Dockerfile for telemetrykeeper services - if config.Spec.TelemetryKeeper.Spec.Enabled { + if config.Spec.TelemetryKeeper.Spec.IsEnabled() { dockerfileBuf := bytes.NewBuffer(nil) err := telemetryKeeperDockerfileTemplate.Execute(dockerfileBuf, config) if err != nil { - return nil, fmt.Errorf("failed to execute dockerfile keeper template: %w", err) + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to execute dockerfile keeper template") } - dockerfileMaterial := types.NewTextMaterial(dockerfileBuf.Bytes(), filepath.Join(configsDir, "telemetrykeeper/Dockerfile")) + dockerfileMaterial := domain.NewBlobMaterial(dockerfileBuf.Bytes(), filepath.Join(configsDir, "telemetrykeeper/Dockerfile")) materials = append(materials, dockerfileMaterial) // Add telemetrykeeper config files (for dockerfile to copy) for filename, content := range config.Spec.TelemetryKeeper.Spec.Config.Data { - material, err := types.NewYAMLMaterial([]byte(content), filepath.Join(configsDir, fmt.Sprintf("telemetrykeeper/keeper.d/%s", filename))) + material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(configsDir, fmt.Sprintf("telemetrykeeper/keeper.d/%s", filename))) if err != nil { - return nil, fmt.Errorf("failed to create telemetrykeeper config material: %w", err) + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to create telemetrykeeper config material") } materials = append(materials, material) } } // Add Dockerfile for telemetrystore services - if config.Spec.TelemetryStore.Spec.Enabled { + if config.Spec.TelemetryStore.Spec.IsEnabled() { dockerfileBuf := bytes.NewBuffer(nil) err := telemetryStoreDockerfileTemplate.Execute(dockerfileBuf, config) if err != nil { - return nil, fmt.Errorf("failed to execute dockerfile clickhouse template: %w", err) + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to execute dockerfile clickhouse template") } - dockerfileMaterial := types.NewTextMaterial(dockerfileBuf.Bytes(), filepath.Join(configsDir, "telemetrystore/Dockerfile")) + dockerfileMaterial := domain.NewBlobMaterial(dockerfileBuf.Bytes(), filepath.Join(configsDir, "telemetrystore/Dockerfile")) materials = append(materials, dockerfileMaterial) // Add telemetrystore config files (for dockerfile to copy) for filename, content := range config.Spec.TelemetryStore.Spec.Config.Data { - material, err := types.NewYAMLMaterial([]byte(content), filepath.Join(configsDir, fmt.Sprintf("telemetrystore/config.d/%s", filename))) + material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(configsDir, fmt.Sprintf("telemetrystore/config.d/%s", filename))) if err != nil { - return nil, fmt.Errorf("failed to create telemetrystore config material: %w", err) + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to create telemetrystore config material") } materials = append(materials, material) } } // Add Dockerfile for ingester services - if config.Spec.Ingester.Spec.Enabled { + if config.Spec.Ingester.Spec.IsEnabled() { dockerfileBuf := bytes.NewBuffer(nil) err := ingesterDockerfileTemplate.Execute(dockerfileBuf, config) if err != nil { - return nil, fmt.Errorf("failed to execute dockerfile otel template: %w", err) + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to execute dockerfile otel template") } - dockerfileMaterial := types.NewTextMaterial(dockerfileBuf.Bytes(), filepath.Join(configsDir, "ingester/Dockerfile")) + dockerfileMaterial := domain.NewBlobMaterial(dockerfileBuf.Bytes(), filepath.Join(configsDir, "ingester/Dockerfile")) materials = append(materials, dockerfileMaterial) for filename, content := range config.Spec.Ingester.Spec.Config.Data { - material, err := types.NewYAMLMaterial([]byte(content), filepath.Join(configsDir, fmt.Sprintf("ingester/%s", filename))) + material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(configsDir, fmt.Sprintf("ingester/%s", filename))) if err != nil { - return nil, fmt.Errorf("failed to create ingester config material: %w", err) + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to create ingester config material") } materials = append(materials, material) } @@ -109,7 +109,7 @@ func (c *renderCasting) Forge(ctx context.Context, config v1alpha1.Casting, pour return materials, nil } -func (c *renderCasting) Cast(ctx context.Context, config v1alpha1.Casting, poursPath string) error { +func (c *renderCasting) Cast(ctx context.Context, config installation.Casting, poursPath string) error { c.logger.InfoContext(ctx, "Please run 'forge' first to generate the Render Casting", slog.String("pours_path", poursPath)) c.logger.InfoContext(ctx, "After forging, deploy render.yaml to Render using Infrastructure as Code", @@ -117,11 +117,11 @@ func (c *renderCasting) Cast(ctx context.Context, config v1alpha1.Casting, pours return nil } -func getRenderMaterial(config *v1alpha1.Casting, path string) (types.Material, error) { +func getRenderMaterial(config *installation.Casting, path string) (domain.StructuredMaterial, error) { buf := bytes.NewBuffer(nil) err := renderYAMLTemplate.Execute(buf, config) if err != nil { - return types.Material{}, fmt.Errorf("failed to execute render yaml template: %w", err) + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to execute render yaml template") } - return types.NewYAMLMaterial(buf.Bytes(), path) + return domain.NewYAMLMaterial(buf.Bytes(), path) } diff --git a/internal/casting/rendercasting/embed.go b/internal/casting/rendercasting/embed.go index bd0338d..9578e26 100644 --- a/internal/casting/rendercasting/embed.go +++ b/internal/casting/rendercasting/embed.go @@ -10,8 +10,8 @@ import ( var templates embed.FS var ( - renderYAMLTemplate *types.Template = types.MustNewTemplateFromFS(templates, "templates/render.yaml.gotmpl", types.FormatYAML) - telemetryKeeperDockerfileTemplate *types.Template = types.MustNewTemplateFromFS(templates, "templates/Dockerfile.clickhousekeeper.telemetrykeeper.v2556.gotmpl", types.FormatText) - telemetryStoreDockerfileTemplate *types.Template = types.MustNewTemplateFromFS(templates, "templates/Dockerfile.clickhouse.telemetrystore.v2556.gotmpl", types.FormatText) - ingesterDockerfileTemplate *types.Template = types.MustNewTemplateFromFS(templates, "templates/Dockerfile.ingester.gotmpl", types.FormatText) + renderYAMLTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/render.yaml.gotmpl", domain.FormatYAML) + telemetryKeeperDockerfileTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/Dockerfile.clickhousekeeper.telemetrykeeper.v2556.gotmpl", domain.FormatText) + telemetryStoreDockerfileTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/Dockerfile.clickhouse.telemetrystore.v2556.gotmpl", domain.FormatText) + ingesterDockerfileTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/Dockerfile.ingester.gotmpl", domain.FormatText) ) diff --git a/internal/casting/rendercasting/embed_test.go b/internal/casting/rendercasting/embed_test.go index 769df23..ae99e72 100644 --- a/internal/casting/rendercasting/embed_test.go +++ b/internal/casting/rendercasting/embed_test.go @@ -9,7 +9,7 @@ import ( ) func TestNotEmptyAndValid(t *testing.T) { - serviceTemplates := map[string]*types.Template{ + serviceTemplates := map[string]*domain.Template{ "telemetryKeeperDockerfileTemplate": telemetryKeeperDockerfileTemplate, "telemetryStoreDockerfileTemplate": telemetryStoreDockerfileTemplate, "ingesterDockerfileTemplate": ingesterDockerfileTemplate, diff --git a/internal/casting/rendercasting/enricher.go b/internal/casting/rendercasting/enricher.go index 1277c85..9c690d5 100644 --- a/internal/casting/rendercasting/enricher.go +++ b/internal/casting/rendercasting/enricher.go @@ -2,7 +2,6 @@ package rendercasting import ( "context" - "fmt" "strings" "github.com/hanzoai/o11y-foundry/api/v1alpha1" @@ -13,32 +12,32 @@ import ( var _ molding.MoldingEnricher = (*renderMoldingEnricher)(nil) type renderMoldingEnricher struct { - material types.Material + material domain.StructuredMaterial } -func newRenderMoldingEnricher(config *v1alpha1.Casting) (*renderMoldingEnricher, error) { +func newRenderMoldingEnricher(config *installation.Casting) (*renderMoldingEnricher, error) { material, err := getRenderMaterial(config, "render.yaml") if err != nil { - return nil, fmt.Errorf("failed to get render yaml material: %w", err) + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to get render yaml material") } return &renderMoldingEnricher{material: material}, nil } -func (enricher *renderMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *v1alpha1.Casting) error { +func (enricher *renderMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *installation.Casting) error { switch kind { case v1alpha1.MoldingKindTelemetryStore: // Get telemetrystore service names serviceNames, err := enricher.material.GetStringSlice("services.#.name") if err != nil { - return fmt.Errorf("failed to get telemetrystore service names: %w", err) + return errors.Wrapf(err, errors.TypeInternal, "failed to get telemetrystore service names") } var addrs []string var storeServiceNames []string for _, serviceName := range serviceNames { if strings.Contains(serviceName, "telemetrystore") && !strings.Contains(serviceName, "migrator") { - addrs = append(addrs, types.FormatAddress("tcp", serviceName, 9000)) + addrs = append(addrs, domain.MustNewAddress("tcp", serviceName, 9000).String()) storeServiceNames = append(storeServiceNames, serviceName) } } @@ -54,7 +53,7 @@ func (enricher *renderMoldingEnricher) EnrichStatus(ctx context.Context, kind v1 // Get telemetrystore service names serviceNames, err := enricher.material.GetStringSlice("services.#.name") if err != nil { - return fmt.Errorf("failed to get telemetrystore service names: %w", err) + return errors.Wrapf(err, errors.TypeInternal, "failed to get telemetrystore service names") } var apiServerAddr []string @@ -72,7 +71,7 @@ func (enricher *renderMoldingEnricher) EnrichStatus(ctx context.Context, kind v1 // Get telemetrykeeper service names serviceNames, err := enricher.material.GetStringSlice("services.#.name") if err != nil { - return fmt.Errorf("failed to get telemetrykeeper service names: %w", err) + return errors.Wrapf(err, errors.TypeInternal, "failed to get telemetrykeeper service names") } var addrsClient []string @@ -80,8 +79,8 @@ func (enricher *renderMoldingEnricher) EnrichStatus(ctx context.Context, kind v1 var keeperServiceNames []string for _, serviceName := range serviceNames { if strings.Contains(serviceName, "telemetrykeeper") { - addrsClient = append(addrsClient, types.FormatAddress("tcp", serviceName, 9181)) - addrsRaft = append(addrsRaft, types.FormatAddress("tcp", serviceName, 9234)) + addrsClient = append(addrsClient, domain.MustNewAddress("tcp", serviceName, 9181).String()) + addrsRaft = append(addrsRaft, domain.MustNewAddress("tcp", serviceName, 9234).String()) keeperServiceNames = append(keeperServiceNames, serviceName) } } @@ -98,13 +97,13 @@ func (enricher *renderMoldingEnricher) EnrichStatus(ctx context.Context, kind v1 // Get ingester service names serviceNames, err := enricher.material.GetStringSlice("services.#.name") if err != nil { - return fmt.Errorf("failed to get ingester service names: %w", err) + return errors.Wrapf(err, errors.TypeInternal, "failed to get ingester service names") } var addrs []string for _, serviceName := range serviceNames { if strings.Contains(serviceName, "ingester") { - addrs = append(addrs, types.FormatAddress("tcp", serviceName, 4318)) + addrs = append(addrs, domain.MustNewAddress("tcp", serviceName, 4318).String()) } } config.Spec.Ingester.Status.Addresses.OTLP = addrs diff --git a/internal/casting/rendercasting/templates/Dockerfile.clickhouse.telemetrystore.v2556.gotmpl b/internal/casting/rendercasting/templates/Dockerfile.clickhouse.telemetrystore.v2556.gotmpl index ece565f..77e0a4d 100644 --- a/internal/casting/rendercasting/templates/Dockerfile.clickhouse.telemetrystore.v2556.gotmpl +++ b/internal/casting/rendercasting/templates/Dockerfile.clickhouse.telemetrystore.v2556.gotmpl @@ -29,4 +29,28 @@ EOF RUN chmod +x /tmp/install-histogram-quantile.sh \ && /tmp/install-histogram-quantile.sh \ - && rm /tmp/install-histogram-quantile.sh \ No newline at end of file + && rm /tmp/install-histogram-quantile.sh + +# Create wrapper entrypoint to alias the right per-node config as config.yaml +RUN cat > /foundry-entrypoint.sh <<'EOF' +#!/bin/bash +set -euo pipefail + +# Extract shard-replica indices from service name (e.g., signoz-telemetrystore-clickhouse-0-1) +INDICES=$(echo "${RENDER_SERVICE_NAME:-}" | grep -oE "[0-9]+-[0-9]+$" || echo "0-0") +CONFIG_SRC="/etc/clickhouse-server/config-${INDICES}.yaml" + +if [ ! -f "$CONFIG_SRC" ]; then + echo "Config $CONFIG_SRC not found, falling back to config-0-0.yaml" + CONFIG_SRC="/etc/clickhouse-server/config-0-0.yaml" +fi + +ln -sf "$CONFIG_SRC" /etc/clickhouse-server/config.yaml +echo "ClickHouse starting with config: $CONFIG_SRC (aliased as config.yaml)" + +exec /entrypoint.sh "$@" +EOF + +RUN chmod +x /foundry-entrypoint.sh + +ENTRYPOINT ["/foundry-entrypoint.sh"] \ No newline at end of file diff --git a/internal/casting/rendercasting/templates/render.yaml.gotmpl b/internal/casting/rendercasting/templates/render.yaml.gotmpl index d48d46a..5bb8ef8 100644 --- a/internal/casting/rendercasting/templates/render.yaml.gotmpl +++ b/internal/casting/rendercasting/templates/render.yaml.gotmpl @@ -103,6 +103,7 @@ services: {{ end }} {{ end }} {{ end }} + {{ end }} {{ if $.Spec.TelemetryStore.Spec.Enabled }} - name: {{ $.Metadata.Name}}-schema-migrator @@ -116,6 +117,10 @@ services: - key: CLICKHOUSE_DSN value: {{ index $.Spec.O11y.Spec.Env "HANZO_TELEMETRYSTORE_CLICKHOUSE_DSN" }} {{ end }} + {{ range $key, $value := $.Spec.Ingester.Spec.Env }} + - key: {{ $key }} + value: {{ $value }} + {{ end }} {{ if index $.Spec.TelemetryStore.Status.Extras "service_names" }} {{ range $index, $serviceName := splitList "," (index $.Spec.TelemetryStore.Status.Extras "service_names") }} - key: DEPENDENCY_LINK_{{ $index }} @@ -142,4 +147,4 @@ databases: user: {{ index $.Spec.MetaStore.Spec.Env "POSTGRES_USER" }} {{ end }} ipAllowList: [] -{{ end }} \ No newline at end of file +{{ end }} diff --git a/internal/casting/systemdcasting/casting.go b/internal/casting/systemdcasting/casting.go index 574c8cf..e80c300 100644 --- a/internal/casting/systemdcasting/casting.go +++ b/internal/casting/systemdcasting/casting.go @@ -1,7 +1,6 @@ package systemdcasting import ( - "bytes" "context" "fmt" "log/slog" @@ -22,21 +21,17 @@ import ( const svcSuffix = ".service" -const ( - serviceStartTimeout = 2 * time.Minute -) - var _ rootcasting.Casting = (*systemdCasting)(nil) type systemdCasting struct { logger *slog.Logger - castings []*types.Template + castings []*domain.Template } func New(logger *slog.Logger) *systemdCasting { return &systemdCasting{ logger: logger, - castings: []*types.Template{ + castings: []*domain.Template{ telemetryKeeperServiceTemplate, telemetryStoreServiceTemplate, metaStoreServiceTemplate, @@ -47,34 +42,34 @@ func New(logger *slog.Logger) *systemdCasting { } } -func (c *systemdCasting) Enricher(ctx context.Context, config *v1alpha1.Casting) (molding.MoldingEnricher, error) { +func (c *systemdCasting) Enricher(ctx context.Context, config *installation.Casting) (molding.MoldingEnricher, error) { return newLinuxMoldingEnricher(config), nil } -func (c *systemdCasting) Forge(ctx context.Context, cfg v1alpha1.Casting, poursPath string) ([]types.Material, error) { - var materials []types.Material +func (c *systemdCasting) Forge(ctx context.Context, cfg installation.Casting, poursPath string) ([]domain.Material, error) { + var materials []domain.Material for _, tmpl := range c.castings { - m, err := c.forgeCasting(tmpl, &cfg, poursPath) + m, err := c.forgeCasting(tmpl, &cfg) if err != nil { - return nil, fmt.Errorf("failed to forge: %w", err) + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to forge") } materials = append(materials, m...) } return materials, nil } -func (c *systemdCasting) Cast(ctx context.Context, config v1alpha1.Casting, poursPath string) error { +func (c *systemdCasting) Cast(ctx context.Context, config installation.Casting, poursPath string) error { c.logger.InfoContext(ctx, "Starting systemd service installation", slog.String("pours_path", poursPath)) runctx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() // Discover and prepare services - serviceMap, err := c.discoverAndPrepareServices(runctx, poursPath) + services, err := c.discoverAndPrepareServices(runctx, poursPath) if err != nil { return err } - if serviceMap == nil { + if len(services) == 0 { c.logger.WarnContext(runctx, "No service files found in pours directory") return nil } @@ -84,14 +79,22 @@ func (c *systemdCasting) Cast(ctx context.Context, config v1alpha1.Casting, pour return err } - if config.Spec.MetaStore.Spec.Enabled { - if err := c.initializePostgres(ctx, &config); err != nil { - return err + if config.Spec.MetaStore.Spec.IsEnabled() { + switch config.Spec.MetaStore.Kind { + case installation.MetaStoreKindPostgres: + if err := c.initializePostgres(ctx, &config); err != nil { + return err + } + case installation.MetaStoreKindSQLite: + if err := os.MkdirAll("/var/lib/signoz", 0755); err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to create sqlite data directory") + } + _ = c.execCommand(ctx, "chown", "-R", "signoz:signoz", "/var/lib/signoz") } } // Start all services - systemd dependencies handle ordering - if err := c.startAllServices(runctx, serviceMap); err != nil { + if err := c.startAllServices(runctx, services); err != nil { return err } @@ -99,58 +102,65 @@ func (c *systemdCasting) Cast(ctx context.Context, config v1alpha1.Casting, pour return nil } -func (c *systemdCasting) forgeCasting(tmpl *types.Template, cfg *v1alpha1.Casting, poursPath string) ([]types.Material, error) { +func (c *systemdCasting) forgeCasting(tmpl *domain.Template, cfg *installation.Casting) ([]domain.Material, error) { switch tmpl { case o11yServiceTemplate: return c.forgeO11y(tmpl, cfg) case metaStoreServiceTemplate: - return c.forgeMetaStore(tmpl, cfg, poursPath) + if !cfg.Spec.MetaStore.Spec.IsEnabled() { + return nil, nil + } + return c.forgeMetaStore(tmpl, cfg) case ingesterServiceTemplate: - return c.forgeIngester(tmpl, cfg, poursPath) + if !cfg.Spec.Ingester.Spec.IsEnabled() { + return nil, nil + } + return c.forgeIngester(tmpl, cfg) case telemetryStoreServiceTemplate: - return c.forgeTelemetryStore(tmpl, cfg, poursPath) + if !cfg.Spec.TelemetryStore.Spec.IsEnabled() { + return nil, nil + } + return c.forgeTelemetryStore(tmpl, cfg) case telemetryKeeperServiceTemplate: - return c.forgeTelemetryKeeper(tmpl, cfg, poursPath) + if !cfg.Spec.TelemetryKeeper.Spec.IsEnabled() { + return nil, nil + } + return c.forgeTelemetryKeeper(tmpl, cfg) case telemetryStoreMigratorServiceTemplate: + if !cfg.Spec.TelemetryStore.Spec.IsEnabled() { + return nil, nil + } return c.forgeMigrator(tmpl, cfg) default: return nil, nil } } -// --- Forge Handlers --- - -func (c *systemdCasting) forgeIngester(tmpl *types.Template, cfg *v1alpha1.Casting, poursPath string) ([]types.Material, error) { +func (c *systemdCasting) forgeIngester(tmpl *domain.Template, cfg *installation.Casting) ([]domain.Material, error) { spec := &cfg.Spec.Ingester - if !spec.Spec.Enabled { - return nil, nil - } - if spec.Status.Config.Data == nil { - return nil, fmt.Errorf("no config molded for %s", v1alpha1.MoldingKindIngester) - } - // Initialize status extras if spec.Status.Extras == nil { spec.Status.Extras = make(map[string]string) } - - // Create config materials - mats, err := c.configMaterials(spec.Status.Config.Data, "ingester") - if err != nil { - return nil, err - } - - // Set extras for template - spec.Status.Extras["cfgPath"] = "configs/ingester/ingester.yaml" - spec.Status.Extras["cfgOpampPath"] = "configs/ingester/opamp.yaml" + spec.Status.Extras["cfgPath"] = filepath.Join("ingester", "ingester.yaml") + spec.Status.Extras["cfgOpampPath"] = filepath.Join("ingester", "opamp.yaml") spec.Status.Extras["workingDir"] = "/opt/ingester" - // Create service material + var materials []domain.Material + svcMat, err := c.renderTemplate(tmpl, cfg, cfg.Metadata.Name+"-ingester"+svcSuffix) if err != nil { return nil, err } - return append(mats, svcMat), nil + materials = append(materials, svcMat) + + cfgMats, err := c.configMaterials(spec.Spec.Config.Data, "ingester", "") + if err != nil { + return nil, err + } + materials = append(materials, cfgMats...) + + return materials, nil } func (c *systemdCasting) forgeO11y(tmpl *types.Template, cfg *v1alpha1.Casting) ([]types.Material, error) { @@ -159,7 +169,6 @@ func (c *systemdCasting) forgeO11y(tmpl *types.Template, cfg *v1alpha1.Casting) return nil, nil } - // Initialize status maps if spec.Status.Extras == nil { spec.Status.Extras = make(map[string]string) } @@ -169,45 +178,41 @@ func (c *systemdCasting) forgeO11y(tmpl *types.Template, cfg *v1alpha1.Casting) spec.Status.Extras["workingDir"] = "/opt/o11y" - // Create service material - svcMat, err := c.renderTemplate(tmpl, cfg, prefix+svcSuffix) + var materials []domain.Material + + svcMat, err := c.renderTemplate(tmpl, cfg, cfg.Metadata.Name+"-signoz"+svcSuffix) if err != nil { return nil, err } - return []types.Material{svcMat}, nil + materials = append(materials, svcMat) + + return materials, nil } -func (c *systemdCasting) forgeMetaStore(tmpl *types.Template, cfg *v1alpha1.Casting, poursPath string) ([]types.Material, error) { +func (c *systemdCasting) forgeMetaStore(tmpl *domain.Template, cfg *installation.Casting) ([]domain.Material, error) { spec := &cfg.Spec.MetaStore - if !spec.Spec.Enabled { - return nil, nil - } - // Initialize status extras if spec.Status.Extras == nil { spec.Status.Extras = make(map[string]string) } - // Create env material - prefix := fmt.Sprintf("%s-metastore-%s", cfg.Metadata.Name, spec.Kind.String()) - // Create service material - svcMat, err := c.renderTemplate(tmpl, cfg, prefix+svcSuffix) - if err != nil { - return nil, err + var materials []domain.Material + + switch spec.Kind { + case installation.MetaStoreKindPostgres: + svcMat, err := c.renderTemplate(tmpl, cfg, fmt.Sprintf("%s-metastore-%s%s", cfg.Metadata.Name, spec.Kind.String(), svcSuffix)) + if err != nil { + return nil, err + } + materials = append(materials, svcMat) } - return []types.Material{svcMat}, nil + + return materials, nil } -func (c *systemdCasting) forgeTelemetryStore(tmpl *types.Template, cfg *v1alpha1.Casting, poursPath string) ([]types.Material, error) { +func (c *systemdCasting) forgeTelemetryStore(tmpl *domain.Template, cfg *installation.Casting) ([]domain.Material, error) { spec := &cfg.Spec.TelemetryStore - if !spec.Spec.Enabled { - return nil, nil - } - if spec.Status.Config.Data == nil { - return nil, fmt.Errorf("no config molded for %s", v1alpha1.MoldingKindTelemetryStore) - } - // Initialize status extras if spec.Status.Extras == nil { spec.Status.Extras = make(map[string]string) } @@ -216,36 +221,30 @@ func (c *systemdCasting) forgeTelemetryStore(tmpl *types.Template, cfg *v1alpha1 reps := max(1, *spec.Spec.Cluster.Replicas+1) shards := max(1, *spec.Spec.Cluster.Shards) - // Create config materials - mats, err := c.configMaterials(spec.Status.Config.Data, "telemetrystore") - if err != nil { - return nil, err - } + var materials []domain.Material - // Create service materials for each shard/replica for s := range shards { for r := range reps { - svcName := fmt.Sprintf("%s-telemetrystore-%s-%d-%d%s", cfg.Metadata.Name, kind, s, r, svcSuffix) - svcMat, err := c.renderTemplate(tmpl, cfg, svcName) + svcMat, err := c.renderTemplate(tmpl, cfg, fmt.Sprintf("%s-telemetrystore-%s-%d-%d%s", cfg.Metadata.Name, kind, s, r, svcSuffix)) if err != nil { return nil, err } - mats = append(mats, svcMat) + materials = append(materials, svcMat) } } - return mats, nil + + cfgMats, err := c.configMaterials(spec.Spec.Config.Data, "telemetrystore", kind) + if err != nil { + return nil, err + } + materials = append(materials, cfgMats...) + + return materials, nil } -func (c *systemdCasting) forgeTelemetryKeeper(tmpl *types.Template, cfg *v1alpha1.Casting, poursPath string) ([]types.Material, error) { +func (c *systemdCasting) forgeTelemetryKeeper(tmpl *domain.Template, cfg *installation.Casting) ([]domain.Material, error) { spec := &cfg.Spec.TelemetryKeeper - if !spec.Spec.Enabled { - return nil, nil - } - if spec.Status.Config.Data == nil { - return nil, fmt.Errorf("no config molded for %s", v1alpha1.MoldingKindTelemetryKeeper) - } - // Initialize status extras if spec.Status.Extras == nil { spec.Status.Extras = make(map[string]string) } @@ -253,63 +252,58 @@ func (c *systemdCasting) forgeTelemetryKeeper(tmpl *types.Template, cfg *v1alpha kind := spec.Kind.String() reps := max(1, *spec.Spec.Cluster.Replicas) - // Create config materials - mats, err := c.configMaterials(spec.Status.Config.Data, "telemetrykeeper") + // Config materials are created first because cfgPath extra is derived from them + cfgMats, err := c.configMaterials(spec.Spec.Config.Data, "telemetrykeeper", kind) if err != nil { return nil, err } + if len(cfgMats) > 0 { + spec.Status.Extras["cfgPath"] = filepath.Join("/etc/clickhouse-keeper/", filepath.Base(cfgMats[0].Path())) + } - // Set config path for template - spec.Status.Extras["cfgPath"] = filepath.Join("/etc/clickhouse-keeper/", filepath.Base(mats[0].Path())) + var materials []domain.Material - // Create service materials for each replica for r := range reps { - svcName := fmt.Sprintf("%s-telemetrykeeper-%s-%d%s", cfg.Metadata.Name, kind, r, svcSuffix) - svcMat, err := c.renderTemplate(tmpl, cfg, svcName) + svcMat, err := c.renderTemplate(tmpl, cfg, fmt.Sprintf("%s-telemetrykeeper-%s-%d%s", cfg.Metadata.Name, kind, r, svcSuffix)) if err != nil { return nil, err } - mats = append(mats, svcMat) + materials = append(materials, svcMat) } - return mats, nil + + materials = append(materials, cfgMats...) + + return materials, nil } -func (c *systemdCasting) forgeMigrator(tmpl *types.Template, cfg *v1alpha1.Casting) ([]types.Material, error) { - spec := &cfg.Spec.TelemetryStore - if !spec.Spec.Enabled { - return nil, nil - } +func (c *systemdCasting) forgeMigrator(tmpl *domain.Template, cfg *installation.Casting) ([]domain.Material, error) { + var materials []domain.Material - // Create service material svcMat, err := c.renderTemplate(tmpl, cfg, cfg.Metadata.Name+"-telemetrystore-migrator"+svcSuffix) if err != nil { return nil, err } - return []types.Material{svcMat}, nil + materials = append(materials, svcMat) + + return materials, nil } -// --- Material Helpers --- - -func (c *systemdCasting) renderTemplate(tmpl *types.Template, cfg *v1alpha1.Casting, path string) (types.Material, error) { - var buf bytes.Buffer - if err := tmpl.Execute(&buf, cfg); err != nil { - return types.Material{}, fmt.Errorf("execute template %s: %w", path, err) - } - return types.NewINIMaterial(buf.Bytes(), filepath.Join(rootcasting.DeploymentDir, path)) -} - -func (c *systemdCasting) configMaterials(data map[string]string, path string) ([]types.Material, error) { - mats := make([]types.Material, 0, len(data)) - for file, content := range data { - m, err := types.NewYAMLMaterial([]byte(content), filepath.Join(rootcasting.DeploymentDir, "configs/", path, file)) +func (c *systemdCasting) configMaterials(data map[string]string, component string, kind string) ([]domain.Material, error) { + mats := make([]domain.Material, 0, len(data)) + for filename, content := range data { + m, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(rootcasting.DeploymentDir, component, kind, filename)) if err != nil { - return nil, fmt.Errorf("failed to create config material %s: %w", file, err) + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to create %s config material %s", component, filename) } mats = append(mats, m) } return mats, nil } +func (c *systemdCasting) renderTemplate(tmpl *domain.Template, cfg *installation.Casting, path string) (domain.Material, error) { + return tmpl.Render(cfg, filepath.Join(rootcasting.DeploymentDir, path)) +} + // execCommand executes a command and returns an error if it fails. func (c *systemdCasting) execCommand(ctx context.Context, name string, args ...string) error { cmd := exec.CommandContext(ctx, name, args...) @@ -318,19 +312,18 @@ func (c *systemdCasting) execCommand(ctx context.Context, name string, args ...s return cmd.Run() } -// discoverAndPrepareServices discovers service files, categorizes them, and prepares systemd. -// Returns nil serviceMap if no services found. -func (c *systemdCasting) discoverAndPrepareServices(ctx context.Context, poursPath string) (map[string][]string, error) { +// discoverAndPrepareServices discovers service files in the pours directory and reloads systemd. +func (c *systemdCasting) discoverAndPrepareServices(ctx context.Context, poursPath string) ([]string, error) { deploymentPath := filepath.Join(poursPath, rootcasting.DeploymentDir) entries, err := os.ReadDir(deploymentPath) if err != nil { - return nil, fmt.Errorf("failed to read directory %s: %w", deploymentPath, err) + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to read directory %s", deploymentPath) } serviceMap := map[string][]string{"keeper": {}, "store": {}, "postgres": {}, "o11y": {}, "ingester": {}, "migrator": {}} for _, entry := range entries { - if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".service") { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), svcSuffix) { continue } servicePath := filepath.Join(deploymentPath, entry.Name()) @@ -354,25 +347,17 @@ func (c *systemdCasting) discoverAndPrepareServices(ctx context.Context, poursPa } } - // Check if any services were found - total := 0 - for cat, svcs := range serviceMap { - if len(svcs) > 0 { - c.logger.DebugContext(ctx, "Found services", slog.String("category", cat), slog.Int("count", len(svcs))) - total += len(svcs) - } - } - if total == 0 { - return map[string][]string{}, nil + if len(services) == 0 { + return nil, nil } - // Reload systemd to pick up new service files - c.logger.DebugContext(ctx, "Reloading systemd daemon") + c.logger.DebugContext(ctx, "Found services", slog.Int("count", len(services))) + if err := c.execCommand(ctx, "systemctl", "daemon-reload"); err != nil { - return nil, fmt.Errorf("systemd daemon-reload failed: %w", err) + return nil, errors.Wrapf(err, errors.TypeInternal, "systemd daemon-reload failed") } - return serviceMap, nil + return services, nil } // setupSystemEnvironment creates o11y user, directories, copies configs, and validates binaries. @@ -387,20 +372,22 @@ func (c *systemdCasting) setupSystemEnvironment(ctx context.Context, config *v1a // Setup working directory if err := os.MkdirAll(poursPath, 0755); err != nil { - return fmt.Errorf("failed to create working directory %s: %w", poursPath, err) + return errors.Wrapf(err, errors.TypeInternal, "failed to create working directory %s", poursPath) } _ = c.execCommand(ctx, "chown", "-R", "o11y:o11y", poursPath) // best effort _ = c.execCommand(ctx, "chown", "-R", "o11y:o11y", "/opt/o11y/") // best effort // Copy clickhouse configs to standard locations - if config.Spec.TelemetryStore.Spec.Enabled { - if err := c.copyDir(filepath.Join(poursPath, rootcasting.DeploymentDir, "configs", "telemetrystore"), "/etc/clickhouse-server/"); err != nil { - return fmt.Errorf("failed to copy clickhouse-server configs: %w", err) + if config.Spec.TelemetryStore.Spec.IsEnabled() { + src := filepath.Join(poursPath, rootcasting.DeploymentDir, "telemetrystore", config.Spec.TelemetryStore.Kind.String()) + if err := c.copyDir(src, "/etc/clickhouse-server/"); err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to copy clickhouse-server configs") } } - if config.Spec.TelemetryKeeper.Spec.Enabled { - if err := c.copyDir(filepath.Join(poursPath, rootcasting.DeploymentDir, "configs", "telemetrykeeper"), "/etc/clickhouse-keeper/"); err != nil { - return fmt.Errorf("failed to copy clickhouse-keeper configs: %w", err) + if config.Spec.TelemetryKeeper.Spec.IsEnabled() { + src := filepath.Join(poursPath, rootcasting.DeploymentDir, "telemetrykeeper", config.Spec.TelemetryKeeper.Kind.String()) + if err := c.copyDir(src, "/etc/clickhouse-keeper/"); err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to copy clickhouse-keeper configs") } } @@ -434,10 +421,10 @@ func (c *systemdCasting) copyDir(srcDir, dstDir string) error { // validateBinaries checks if binaries exist at annotation paths. // Only validates if annotations are set; defaults are handled in templates. -func (c *systemdCasting) validateBinaries(config *v1alpha1.Casting) error { +func (c *systemdCasting) validateBinaries(config *installation.Casting) error { annotations := config.Metadata.Annotations if annotations == nil { - return fmt.Errorf("no binary paths found in annotations") + return errors.Newf(errors.TypeInvalidInput, "no binary paths found in annotations") } var missing []string @@ -457,39 +444,35 @@ func (c *systemdCasting) validateBinaries(config *v1alpha1.Casting) error { } if len(missing) > 0 { - return fmt.Errorf("missing binaries: %s - please install before running cast", strings.Join(missing, ", ")) + return errors.Newf(errors.TypeNotFound, "missing binaries: %s - please install before running cast", strings.Join(missing, ", ")) } return nil } // startAllServices enables and starts all discovered services. -// Systemd dependencies ensure proper ordering. -func (c *systemdCasting) startAllServices(ctx context.Context, serviceMap map[string][]string) error { - // Collect all services - var allServices []string - for _, services := range serviceMap { - allServices = append(allServices, services...) - } - - if len(allServices) == 0 { - return nil - } - - // Enable and start all services - systemd will handle ordering via dependencies - for _, svc := range allServices { +// All services are enabled first so that dependency references resolve, +// then started — systemd handles ordering via After=/Requires=. +func (c *systemdCasting) startAllServices(ctx context.Context, services []string) error { + // Enable all services first so dependencies can be resolved + for _, svc := range services { unitName := filepath.Base(svc) - c.logger.DebugContext(ctx, "Enabling service", slog.String("service", unitName), slog.String("path", svc)) - // Use full path for enable - systemctl can work with paths to service files + c.logger.DebugContext(ctx, "Enabling service", slog.String("service", unitName)) if err := c.execCommand(ctx, "systemctl", "enable", svc); err != nil { - return fmt.Errorf("failed to enable service %s: %w", unitName, err) + return errors.Wrapf(err, errors.TypeInternal, "failed to enable service %s", unitName) } + } + + // Reload systemd to pick up all enabled services + if err := c.execCommand(ctx, "systemctl", "daemon-reload"); err != nil { + return errors.Wrapf(err, errors.TypeInternal, "systemd daemon-reload failed") + } + + // Start all services without blocking — systemd dependencies handle ordering + for _, svc := range services { + unitName := filepath.Base(svc) c.logger.InfoContext(ctx, "Starting service", slog.String("service", unitName)) - startCtx, cancel := context.WithTimeout(ctx, serviceStartTimeout) - // Use full path for start as well - err := c.execCommand(startCtx, "systemctl", "start", unitName) - cancel() - if err != nil { - return fmt.Errorf("failed to start service %s: %w", unitName, err) + if err := c.execCommand(ctx, "systemctl", "start", "--no-block", unitName); err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to start service %s", unitName) } } @@ -497,7 +480,7 @@ func (c *systemdCasting) startAllServices(ctx context.Context, serviceMap map[st } // initializePostgres sets up the PostgreSQL data directory. -func (c *systemdCasting) initializePostgres(ctx context.Context, config *v1alpha1.Casting) error { +func (c *systemdCasting) initializePostgres(ctx context.Context, config *installation.Casting) error { pgDataDir := "/usr/local/pgsql/data" pwfile := "/tmp/postgres_pwfile_init" @@ -514,11 +497,11 @@ func (c *systemdCasting) initializePostgres(ctx context.Context, config *v1alpha // Create directories if err := os.MkdirAll(pgDataDir, 0700); err != nil { - return fmt.Errorf("failed to create PostgreSQL data directory: %w", err) + return errors.Wrapf(err, errors.TypeInternal, "failed to create PostgreSQL data directory") } if err := c.execCommand(ctx, "chown", "-R", "postgres:postgres", filepath.Dir(pgDataDir)); err != nil { - return fmt.Errorf("failed to set ownership on PostgreSQL data directory: %w", err) + return errors.Wrapf(err, errors.TypeInternal, "failed to set ownership on PostgreSQL data directory") } // Get credentials @@ -538,7 +521,7 @@ func (c *systemdCasting) initializePostgres(ctx context.Context, config *v1alpha // Create password file if err := os.WriteFile(pwfile, []byte(pgPass+"\n"), 0600); err != nil { - return fmt.Errorf("failed to create password file: %w", err) + return errors.Wrapf(err, errors.TypeInternal, "failed to create password file") } _ = c.execCommand(ctx, "chown", "postgres:postgres", pwfile) @@ -546,7 +529,7 @@ func (c *systemdCasting) initializePostgres(ctx context.Context, config *v1alpha postgresBin := config.Metadata.Annotations["foundry.o11y.hanzo.ai/metastore-postgres-binary-path"] if postgresBin == "" { - return fmt.Errorf("metastore postgres binary is missing in annotations") + return errors.Newf(errors.TypeInvalidInput, "metastore postgres binary is missing in annotations") } postgresBinDir := filepath.Dir(postgresBin) @@ -558,7 +541,7 @@ func (c *systemdCasting) initializePostgres(ctx context.Context, config *v1alpha if err := c.execCommand(ctx, "su", "-", "postgres", "-c", fmt.Sprintf("%s -D %s --username=%s --pwfile=%s", initdbPath, pgDataDir, pgUser, pwfile)); err != nil { c.cleanupPostgresInit(ctx, pgDataDir, pwfile) - return fmt.Errorf("failed to initialize PostgreSQL: %w", err) + return errors.Wrapf(err, errors.TypeInternal, "failed to initialize PostgreSQL") } // Start temp server and create database @@ -566,7 +549,7 @@ func (c *systemdCasting) initializePostgres(ctx context.Context, config *v1alpha if err := c.execCommand(ctx, "su", "-", "postgres", "-c", fmt.Sprintf("%s -D %s -o \"-c listen_addresses=localhost\" -w start", pgCtlPath, pgDataDir)); err != nil { c.cleanupPostgresInit(ctx, pgDataDir, pwfile) - return fmt.Errorf("failed to start temporary postgres: %w", err) + return errors.Wrapf(err, errors.TypeInternal, "failed to start temporary postgres") } // Create database @@ -577,12 +560,12 @@ func (c *systemdCasting) initializePostgres(ctx context.Context, config *v1alpha // Stop temporary PostgreSQL if err := c.execCommand(ctx, "su", "-", "postgres", "-c", fmt.Sprintf("%s -D %s -m fast -w stop", pgCtlPath, pgDataDir)); err != nil { - return fmt.Errorf("failed to stop temporary postgres: %w", err) + return errors.Wrapf(err, errors.TypeInternal, "failed to stop temporary postgres") } // Clean up password file if err := os.Remove(pwfile); err != nil { - return fmt.Errorf("failed to remove password file: %w", err) + return errors.Wrapf(err, errors.TypeInternal, "failed to remove password file") } return nil diff --git a/internal/casting/systemdcasting/embed_test.go b/internal/casting/systemdcasting/embed_test.go index 62e17a3..b80b5a7 100644 --- a/internal/casting/systemdcasting/embed_test.go +++ b/internal/casting/systemdcasting/embed_test.go @@ -9,7 +9,7 @@ import ( ) func TestNotEmptyAndValid(t *testing.T) { - serviceTemplates := map[string]*types.Template{ + serviceTemplates := map[string]*domain.Template{ "telemetryStoreServiceTemplate": telemetryStoreServiceTemplate, "telemetryKeeperServiceTemplate": telemetryKeeperServiceTemplate, "metaStoreServiceTemplate": metaStoreServiceTemplate, diff --git a/internal/casting/systemdcasting/enricher.go b/internal/casting/systemdcasting/enricher.go index 3e55df2..894e31b 100644 --- a/internal/casting/systemdcasting/enricher.go +++ b/internal/casting/systemdcasting/enricher.go @@ -2,7 +2,7 @@ package systemdcasting import ( "context" - "fmt" + "path" "github.com/hanzoai/o11y-foundry/api/v1alpha1" "github.com/hanzoai/o11y-foundry/internal/molding" @@ -19,14 +19,14 @@ const ( ) type linuxMoldingEnricher struct { - materials []types.Material + materials []domain.Material } -func newLinuxMoldingEnricher(_ *v1alpha1.Casting) *linuxMoldingEnricher { - return &linuxMoldingEnricher{materials: []types.Material{}} +func newLinuxMoldingEnricher(_ *installation.Casting) *linuxMoldingEnricher { + return &linuxMoldingEnricher{materials: []domain.Material{}} } -func (e *linuxMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *v1alpha1.Casting) error { +func (e *linuxMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *installation.Casting) error { switch kind { case v1alpha1.MoldingKindTelemetryStore: return e.enrichTelemetryStore(config) @@ -42,7 +42,7 @@ func (e *linuxMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.M return nil } -func (e *linuxMoldingEnricher) enrichTelemetryStore(config *v1alpha1.Casting) error { +func (e *linuxMoldingEnricher) enrichTelemetryStore(config *installation.Casting) error { spec := &config.Spec.TelemetryStore cluster := spec.Spec.Cluster @@ -64,7 +64,7 @@ func (e *linuxMoldingEnricher) enrichTelemetryStore(config *v1alpha1.Casting) er for shard := 0; shard < shards; shard++ { for replica := 0; replica < replicas; replica++ { port := baseTelemetryStoreClusterPort + (shard * replicas) + replica - addresses = append(addresses, types.FormatAddress("tcp", "localhost", port)) + addresses = append(addresses, domain.MustNewAddress("tcp", "localhost", port).String()) } } @@ -72,7 +72,7 @@ func (e *linuxMoldingEnricher) enrichTelemetryStore(config *v1alpha1.Casting) er return nil } -func (e *linuxMoldingEnricher) enrichTelemetryKeeper(config *v1alpha1.Casting) error { +func (e *linuxMoldingEnricher) enrichTelemetryKeeper(config *installation.Casting) error { spec := &config.Spec.TelemetryKeeper cluster := spec.Spec.Cluster @@ -87,8 +87,8 @@ func (e *linuxMoldingEnricher) enrichTelemetryKeeper(config *v1alpha1.Casting) e var clientAddresses, raftAddresses []string for r := 0; r < replicas; r++ { - clientAddresses = append(clientAddresses, types.FormatAddress("tcp", "localhost", baseTelemetryKeeperClientPort+r)) - raftAddresses = append(raftAddresses, types.FormatAddress("tcp", "localhost", baseTelemetryKeeperRaftPort+r)) + clientAddresses = append(clientAddresses, domain.MustNewAddress("tcp", "localhost", baseTelemetryKeeperClientPort+r).String()) + raftAddresses = append(raftAddresses, domain.MustNewAddress("tcp", "localhost", baseTelemetryKeeperRaftPort+r).String()) } config.Spec.TelemetryKeeper.Status.Addresses.Client = clientAddresses @@ -96,19 +96,25 @@ func (e *linuxMoldingEnricher) enrichTelemetryKeeper(config *v1alpha1.Casting) e return nil } -func (e *linuxMoldingEnricher) enrichMetaStore(config *v1alpha1.Casting) error { - dsn := types.FormatAddress("postgres", "localhost", baseMetaStorePostgresPort) - config.Spec.MetaStore.Status.Addresses.DSN = []string{dsn} +func (e *linuxMoldingEnricher) enrichMetaStore(config *installation.Casting) error { + switch config.Spec.MetaStore.Kind { + case installation.MetaStoreKindSQLite: + // SQLite — no addresses or binaries to enrich. + case installation.MetaStoreKindPostgres: + dsn := domain.MustNewAddress("postgres", "localhost", baseMetaStorePostgresPort).String() + config.Spec.MetaStore.Status.Addresses.DSN = []string{dsn} // Get the annotation value metastoreBin := config.Metadata.Annotations["foundry.o11y.hanzo.ai/metastore-postgres-binary-path"] - // If it's missing, apply the default and write it back - if metastoreBin == "" { - metastoreBin = "/usr/bin/postgres" + // If it's missing, apply the default and write it back + if metastoreBin == "" { + metastoreBin = "/usr/bin/postgres" - if config.Metadata.Annotations == nil { - config.Metadata.Annotations = make(map[string]string) + if config.Metadata.Annotations == nil { + config.Metadata.Annotations = make(map[string]string) + } + config.Metadata.Annotations["foundry.signoz.io/metastore-postgres-binary-path"] = metastoreBin } config.Metadata.Annotations["foundry.o11y.hanzo.ai/metastore-postgres-binary-path"] = metastoreBin } @@ -136,12 +142,23 @@ func (e *linuxMoldingEnricher) enrichO11y(config *v1alpha1.Casting) error { config.Metadata.Annotations["foundry.o11y.hanzo.ai/o11y-binary-path"] = o11yBin } + // The binary defaults these to its in-container paths, so point them at the + // extracted tarball tree (binary lives at <root>/bin/signoz). + root := path.Dir(path.Dir(signozBin)) + if config.Spec.Signoz.Status.Env == nil { + config.Spec.Signoz.Status.Env = make(map[string]string) + } + env := config.Spec.Signoz.Status.Env + env["SIGNOZ_WEB_DIRECTORY"] = path.Join(root, "web") + env["SIGNOZ_EMAILING_TEMPLATES_DIRECTORY"] = path.Join(root, "templates", "email") + env["SIGNOZ_ALERTMANAGER_SIGNOZ_TEMPLATES"] = path.Join(root, "templates", "alertmanager", "*.gotmpl") + return nil } -func (e *linuxMoldingEnricher) enrichIngester(config *v1alpha1.Casting) error { +func (e *linuxMoldingEnricher) enrichIngester(config *installation.Casting) error { config.Spec.Ingester.Status.Addresses.OTLP = []string{ - types.FormatAddress("tcp", "localhost", 4317), + domain.MustNewAddress("tcp", "localhost", 4317).String(), } // Get the annotation value @@ -157,5 +174,10 @@ func (e *linuxMoldingEnricher) enrichIngester(config *v1alpha1.Casting) error { config.Metadata.Annotations["foundry.o11y.hanzo.ai/ingester-binary-path"] = ingesterBin } + if config.Spec.Ingester.Status.Env == nil { + config.Spec.Ingester.Status.Env = make(map[string]string) + } + config.Spec.Ingester.Status.Env["SIGNOZ_OTEL_COLLECTOR_TIMEOUT"] = "10m" + return nil } diff --git a/internal/casting/systemdcasting/templates/clickhouse.telemetrystore.v2556.service.gotmpl b/internal/casting/systemdcasting/templates/clickhouse.telemetrystore.v2556.service.gotmpl index cb815fe..9b84092 100644 --- a/internal/casting/systemdcasting/templates/clickhouse.telemetrystore.v2556.service.gotmpl +++ b/internal/casting/systemdcasting/templates/clickhouse.telemetrystore.v2556.service.gotmpl @@ -1,9 +1,8 @@ [Unit] Description=ClickHouse Server (v25.5.6) Requires=network-online.target -After=time-sync.target network-online.target{{ if $.Spec.TelemetryKeeper.Spec.Enabled }} {{ $.Metadata.Name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-0.service{{ end }} -Wants=time-sync.target{{ if $.Spec.TelemetryKeeper.Spec.Enabled }} {{ $.Metadata.Name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-0.service{{ end }} - +After=time-sync.target network-online.target +Wants=time-sync.target [Service] Type=simple @@ -12,10 +11,13 @@ Group=clickhouse Restart=always RestartSec=30 RuntimeDirectory=clickhouse -ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config.yaml --pid-file=/run/clickhouse/clickhouse.pid +ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config-0-0.yaml --pid-file=/run/clickhouse/clickhouse.pid +{{- range $key, $value := $.Spec.TelemetryStore.Spec.Env }} +Environment={{ $key }}={{ $value }} +{{- end }} LimitCORE=infinity LimitNOFILE=500000 CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE CAP_NET_BIND_SERVICE - + [Install] WantedBy=multi-user.target diff --git a/internal/casting/systemdcasting/templates/clickhousekeeper.telemetrykeeper.v2556.service.gotmpl b/internal/casting/systemdcasting/templates/clickhousekeeper.telemetrykeeper.v2556.service.gotmpl index c7e9ec1..b752751 100644 --- a/internal/casting/systemdcasting/templates/clickhousekeeper.telemetrykeeper.v2556.service.gotmpl +++ b/internal/casting/systemdcasting/templates/clickhousekeeper.telemetrykeeper.v2556.service.gotmpl @@ -11,9 +11,12 @@ Group=clickhouse Restart=always RestartSec=30 RuntimeDirectory=clickhouse-keeper -{{ if and $.Spec.TelemetryKeeper.Status.Extras (index $.Spec.TelemetryKeeper.Status.Extras "cfgPath") }} +{{- if and $.Spec.TelemetryKeeper.Status.Extras (index $.Spec.TelemetryKeeper.Status.Extras "cfgPath") }} ExecStart=/usr/bin/clickhouse-keeper --config={{ index $.Spec.TelemetryKeeper.Status.Extras "cfgPath" }} --pid-file=/run/clickhouse-keeper/clickhouse-keeper.pid -{{ end }} +{{- end }} +{{- range $key, $value := $.Spec.TelemetryKeeper.Spec.Env }} +Environment={{ $key }}={{ $value }} +{{- end }} LimitCORE=infinity LimitNOFILE=500000 CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE CAP_NET_BIND_SERVICE diff --git a/internal/casting/systemdcasting/templates/ingester.service.gotmpl b/internal/casting/systemdcasting/templates/ingester.service.gotmpl index 9bbe6a1..8b6c17c 100644 --- a/internal/casting/systemdcasting/templates/ingester.service.gotmpl +++ b/internal/casting/systemdcasting/templates/ingester.service.gotmpl @@ -17,7 +17,14 @@ Restart=on-failure {{- $ingesterBinaryPath := index $.Metadata.Annotations "foundry.o11y.io/ingester-binary-path" }} {{ if and $.Spec.Ingester.Status.Extras (index $.Spec.Ingester.Status.Extras "cfgPath") (index $.Spec.Ingester.Status.Extras "cfgOpampPath") }} ExecStart={{ $ingesterBinaryPath }} --config=%Y/{{ index $.Spec.Ingester.Status.Extras "cfgPath" }} --manager-config=%Y/{{ index $.Spec.Ingester.Status.Extras "cfgOpampPath" }} --copy-path=/var/lib/ingester/ingester.yaml -{{ end }} {{- end }} +{{- end }} +{{- if and $.Spec.TelemetryStore.Spec.Enabled $.Spec.TelemetryStore.Status.Addresses.TCP (index $.Spec.TelemetryStore.Status.Addresses.TCP 0) }} +Environment=SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN={{ index $.Spec.TelemetryStore.Status.Addresses.TCP 0 }} +{{- end }} +{{- range $key, $value := $.Spec.Ingester.Spec.Env }} +Environment={{ $key }}={{ $value }} +{{- end }} + [Install] WantedBy=multi-user.target diff --git a/internal/casting/systemdcasting/templates/postgres.metastore.service.gotmpl b/internal/casting/systemdcasting/templates/postgres.metastore.service.gotmpl index 2a89ca7..b0c92c7 100644 --- a/internal/casting/systemdcasting/templates/postgres.metastore.service.gotmpl +++ b/internal/casting/systemdcasting/templates/postgres.metastore.service.gotmpl @@ -7,8 +7,7 @@ After=network-online.target [Service] Type=notify User=postgres -{{ if $.Spec.MetaStore.Spec.Env }} -{{ range $key, $value := $.Spec.MetaStore.Spec.Env }} +{{- range $key, $value := $.Spec.MetaStore.Spec.Env }} Environment={{ $key }}={{ $value }} {{ end }} {{ end }} @@ -22,4 +21,4 @@ KillSignal=SIGINT TimeoutSec=infinity [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target diff --git a/internal/config/config.go b/internal/config/config.go index 5a719e9..f1c7520 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,13 +7,12 @@ import ( ) type Config interface { - // Gets the v1alpha1 casting configuration from the given path. - // It validates the configuration and returns an error if the configuration is invalid. - GetV1Alpha1(ctx context.Context, path string) (v1alpha1.Casting, error) + // GetV1Alpha1 reads, dispatches, and validates a v1alpha1 casting from disk. + GetV1Alpha1(ctx context.Context, path string) (v1alpha1.Machinery, error) - // Creates the v1alpha1 casting lock file from the given configuration. This will have some support for checksumming in the future. - CreateV1Alpha1Lock(ctx context.Context, config v1alpha1.Casting, path string) error + // CreateV1Alpha1Lock writes the resolved casting to the lock file. + CreateV1Alpha1Lock(ctx context.Context, machinery v1alpha1.Machinery, path string) error - // Gets the v1alpha1 lock file from the given path. This will validate the checksum of the configuration in the future. - GetV1Alpha1Lock(ctx context.Context, path string) (v1alpha1.Casting, error) + // GetV1Alpha1Lock reads the lock file from disk. + GetV1Alpha1Lock(ctx context.Context, path string) (v1alpha1.Machinery, error) } diff --git a/internal/config/yamlconfig/config.go b/internal/config/yamlconfig/config.go index d801713..85162ae 100644 --- a/internal/config/yamlconfig/config.go +++ b/internal/config/yamlconfig/config.go @@ -2,7 +2,7 @@ package yamlconfig import ( "context" - "fmt" + "encoding/json" "os" "path/filepath" @@ -12,60 +12,149 @@ import ( ) type yamlConfig struct { + loaders map[v1alpha1.Kind]loaderFn } +type loaderFn func(bytes []byte, path string) (v1alpha1.Machinery, error) + func New() config.Config { - return &yamlConfig{} + return &yamlConfig{ + loaders: map[v1alpha1.Kind]loaderFn{ + v1alpha1.KindInstallation: loadInstallation, + v1alpha1.KindCollectionAgent: loadCollectionAgent, + }, + } } -func (*yamlConfig) GetV1Alpha1(ctx context.Context, path string) (v1alpha1.Casting, error) { +// GetV1Alpha1 reads, peeks at kind, dispatches to the per-Kind loader, merges +// defaults, validates against the per-Kind schema, and returns the resolved +// casting wrapped as v1alpha1.Machinery. +func (c *yamlConfig) GetV1Alpha1(ctx context.Context, path string) (v1alpha1.Machinery, error) { bytes, err := os.ReadFile(path) if err != nil { - return v1alpha1.Casting{}, fmt.Errorf("failed to read yaml file: %w", err) + return nil, errors.Wrapf(err, errors.TypeNotFound, "failed to read yaml file") } - var casting v1alpha1.Casting - - err = types.UnmarshalYAML(bytes, &casting) + kind, err := peekKind(bytes) if err != nil { - return v1alpha1.Casting{}, fmt.Errorf("failed to unmarshal yaml: %w", err) + return nil, err } - defaultCasting := v1alpha1.DefaultCasting() - // merge overrides into defaults (base) - if err := v1alpha1.Merge(&defaultCasting, &casting); err != nil { - return v1alpha1.Casting{}, fmt.Errorf("failed to merge default casting: %w", err) + load, ok := c.loaders[kind] + if !ok { + return nil, errors.Newf(errors.TypeUnsupported, "unknown casting kind %q", kind) } - - return defaultCasting, nil + return load(bytes, path) } -func (*yamlConfig) CreateV1Alpha1Lock(ctx context.Context, config v1alpha1.Casting, path string) error { - contents, err := types.MarshalYAML(config) - if err != nil { - return fmt.Errorf("failed to marshal yaml: %w", err) +// peekKind decodes only the kind field from raw bytes. Empty or missing kind +// defaults to KindInstallation so existing castings without `kind` keep working. +func peekKind(bytes []byte) (v1alpha1.Kind, error) { + var probe struct { + Kind v1alpha1.Kind `json:"kind" yaml:"kind"` + } + if err := domain.UnmarshalYAML(bytes, &probe); err != nil { + return v1alpha1.Kind{}, errors.Wrapf(err, errors.TypeInvalidInput, "failed to peek kind") + } + if probe.Kind == (v1alpha1.Kind{}) { + return v1alpha1.KindInstallation, nil + } + return probe.Kind, nil +} + +func loadInstallation(bytes []byte, path string) (v1alpha1.Machinery, error) { + var loaded installation.Casting + if err := domain.UnmarshalYAML(bytes, &loaded); err != nil { + return nil, errors.Wrapf(err, errors.TypeInvalidInput, "failed to unmarshal installation casting") } - err = os.WriteFile(filepath.Join(filepath.Dir(path), "casting.yaml.lock"), contents, 0644) + base := installation.Default() + if err := v1alpha1.Merge(base, &loaded); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to merge default installation casting") + } + + contents, err := json.Marshal(base) if err != nil { - return fmt.Errorf("failed to write yaml file: %w", err) + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to marshal installation casting") + } + toValidate := map[string]any{} + if err := json.Unmarshal(contents, &toValidate); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to unmarshal installation casting for validation") + } + + if err := installation.Schema().Validate(toValidate); err != nil { + return nil, errors.Wrapf(err, errors.TypeInvalidInput, "invalid casting file %s", path) + } + + return base, nil +} + +func loadCollectionAgent(bytes []byte, path string) (v1alpha1.Machinery, error) { + var loaded collectionagent.Casting + if err := domain.UnmarshalYAML(bytes, &loaded); err != nil { + return nil, errors.Wrapf(err, errors.TypeInvalidInput, "failed to unmarshal collectionagent casting") + } + + base := collectionagent.Default() + if err := v1alpha1.Merge(base, &loaded); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to merge default collectionagent casting") + } + + contents, err := json.Marshal(base) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to marshal collectionagent casting") + } + toValidate := map[string]any{} + if err := json.Unmarshal(contents, &toValidate); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to unmarshal collectionagent casting for validation") + } + + if err := collectionagent.Schema().Validate(toValidate); err != nil { + return nil, errors.Wrapf(err, errors.TypeInvalidInput, "invalid casting file %s", path) + } + + return base, nil +} + +// CreateV1Alpha1Lock writes the resolved casting to the lock file. +func (*yamlConfig) CreateV1Alpha1Lock(ctx context.Context, machinery v1alpha1.Machinery, path string) error { + contents, err := domain.MarshalYAML(machinery) + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to marshal yaml") + } + + if err := os.WriteFile(filepath.Join(filepath.Dir(path), "casting.yaml.lock"), contents, 0644); err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to write yaml file") } return nil } -func (*yamlConfig) GetV1Alpha1Lock(ctx context.Context, path string) (v1alpha1.Casting, error) { +// GetV1Alpha1Lock reads the lock file and dispatches by kind. +func (*yamlConfig) GetV1Alpha1Lock(ctx context.Context, path string) (v1alpha1.Machinery, error) { bytes, err := os.ReadFile(filepath.Join(filepath.Dir(path), "casting.yaml.lock")) if err != nil { - return v1alpha1.Casting{}, fmt.Errorf("failed to read yaml file: %w", err) + return nil, errors.Wrapf(err, errors.TypeNotFound, "failed to read yaml file") } - var casting v1alpha1.Casting - - err = types.UnmarshalYAML(bytes, &casting) + kind, err := peekKind(bytes) if err != nil { - return v1alpha1.Casting{}, fmt.Errorf("failed to unmarshal yaml: %w", err) + return nil, err } - return casting, nil + switch kind { + case v1alpha1.KindInstallation: + var c installation.Casting + if err := domain.UnmarshalYAML(bytes, &c); err != nil { + return nil, errors.Wrapf(err, errors.TypeInvalidInput, "failed to unmarshal installation casting") + } + return &c, nil + case v1alpha1.KindCollectionAgent: + var c collectionagent.Casting + if err := domain.UnmarshalYAML(bytes, &c); err != nil { + return nil, errors.Wrapf(err, errors.TypeInvalidInput, "failed to unmarshal collectionagent casting") + } + return &c, nil + } + return nil, errors.Newf(errors.TypeUnsupported, "unknown casting kind %q", kind) } diff --git a/internal/config/yamlconfig/config_test.go b/internal/config/yamlconfig/config_test.go new file mode 100644 index 0000000..909360d --- /dev/null +++ b/internal/config/yamlconfig/config_test.go @@ -0,0 +1,326 @@ +package yamlconfig + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/installation" + "github.com/signoz/foundry/internal/domain" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetV1Alpha1(t *testing.T) { + testCases := []struct { + name string + input string + assert func(t *testing.T, casting installation.Casting) + }{ + { + name: "Defaults", + input: ` +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + mode: docker + flavor: compose +`, + assert: func(t *testing.T, casting installation.Casting) { + // All moldings should be enabled by default + assert.True(t, *casting.Spec.Signoz.Spec.Enabled) + assert.True(t, *casting.Spec.TelemetryStore.Spec.Enabled) + assert.True(t, *casting.Spec.TelemetryKeeper.Spec.Enabled) + assert.True(t, *casting.Spec.MetaStore.Spec.Enabled) + assert.True(t, *casting.Spec.Ingester.Spec.Enabled) + }, + }, + { + name: "DisableMetaStore", + input: ` +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + mode: docker + flavor: compose + metastore: + spec: + enabled: false +`, + assert: func(t *testing.T, casting installation.Casting) { + assert.False(t, *casting.Spec.MetaStore.Spec.Enabled) + // Other moldings should remain enabled + assert.True(t, *casting.Spec.Signoz.Spec.Enabled) + assert.True(t, *casting.Spec.TelemetryStore.Spec.Enabled) + assert.True(t, *casting.Spec.TelemetryKeeper.Spec.Enabled) + assert.True(t, *casting.Spec.Ingester.Spec.Enabled) + }, + }, + { + name: "DisableSignoz", + input: ` +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + mode: docker + flavor: compose + signoz: + spec: + enabled: false +`, + assert: func(t *testing.T, casting installation.Casting) { + assert.False(t, *casting.Spec.Signoz.Spec.Enabled) + assert.True(t, *casting.Spec.TelemetryStore.Spec.Enabled) + assert.True(t, *casting.Spec.TelemetryKeeper.Spec.Enabled) + assert.True(t, *casting.Spec.MetaStore.Spec.Enabled) + assert.True(t, *casting.Spec.Ingester.Spec.Enabled) + }, + }, + { + name: "DisableIngester", + input: ` +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + mode: docker + flavor: compose + ingester: + spec: + enabled: false +`, + assert: func(t *testing.T, casting installation.Casting) { + assert.False(t, *casting.Spec.Ingester.Spec.Enabled) + assert.True(t, *casting.Spec.Signoz.Spec.Enabled) + assert.True(t, *casting.Spec.TelemetryStore.Spec.Enabled) + assert.True(t, *casting.Spec.TelemetryKeeper.Spec.Enabled) + assert.True(t, *casting.Spec.MetaStore.Spec.Enabled) + }, + }, + { + name: "DisableTelemetryStore", + input: ` +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + mode: docker + flavor: compose + telemetrystore: + spec: + enabled: false +`, + assert: func(t *testing.T, casting installation.Casting) { + assert.False(t, *casting.Spec.TelemetryStore.Spec.Enabled) + assert.True(t, *casting.Spec.Signoz.Spec.Enabled) + assert.True(t, *casting.Spec.TelemetryKeeper.Spec.Enabled) + assert.True(t, *casting.Spec.MetaStore.Spec.Enabled) + assert.True(t, *casting.Spec.Ingester.Spec.Enabled) + }, + }, + { + name: "DisableTelemetryKeeper", + input: ` +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + mode: docker + flavor: compose + telemetrykeeper: + spec: + enabled: false +`, + assert: func(t *testing.T, casting installation.Casting) { + assert.False(t, *casting.Spec.TelemetryKeeper.Spec.Enabled) + assert.True(t, *casting.Spec.Signoz.Spec.Enabled) + assert.True(t, *casting.Spec.TelemetryStore.Spec.Enabled) + assert.True(t, *casting.Spec.MetaStore.Spec.Enabled) + assert.True(t, *casting.Spec.Ingester.Spec.Enabled) + }, + }, + { + name: "DisableMultipleMoldings", + input: ` +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + mode: docker + flavor: compose + metastore: + spec: + enabled: false + telemetrykeeper: + spec: + enabled: false +`, + assert: func(t *testing.T, casting installation.Casting) { + assert.False(t, *casting.Spec.MetaStore.Spec.Enabled) + assert.False(t, *casting.Spec.TelemetryKeeper.Spec.Enabled) + assert.True(t, *casting.Spec.Signoz.Spec.Enabled) + assert.True(t, *casting.Spec.TelemetryStore.Spec.Enabled) + assert.True(t, *casting.Spec.Ingester.Spec.Enabled) + }, + }, + { + name: "DisabledWithOtherFields", + input: ` +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + mode: docker + flavor: compose + metastore: + spec: + enabled: false + image: custom:1.0 +`, + assert: func(t *testing.T, casting installation.Casting) { + assert.False(t, *casting.Spec.MetaStore.Spec.Enabled) + assert.Equal(t, "custom:1.0", casting.Spec.MetaStore.Spec.Image) + }, + }, + { + name: "ExplicitEnabledTrue", + input: ` +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + mode: docker + flavor: compose + metastore: + spec: + enabled: true +`, + assert: func(t *testing.T, casting installation.Casting) { + assert.True(t, *casting.Spec.MetaStore.Spec.Enabled) + }, + }, + { + name: "OverrideImageKeepsEnabledDefault", + input: ` +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + mode: docker + flavor: compose + metastore: + spec: + image: postgres:15 +`, + assert: func(t *testing.T, casting installation.Casting) { + // Enabled should remain true (default) when only image is overridden + assert.True(t, *casting.Spec.MetaStore.Spec.Enabled) + assert.Equal(t, "postgres:15", casting.Spec.MetaStore.Spec.Image) + }, + }, + { + name: "OverrideVersion", + input: ` +apiVersion: v1alpha1 +metadata: + name: signoz +spec: + deployment: + mode: docker + flavor: compose + telemetrystore: + spec: + version: "24.8" +`, + assert: func(t *testing.T, casting installation.Casting) { + assert.True(t, *casting.Spec.TelemetryStore.Spec.Enabled) + assert.Equal(t, "24.8", casting.Spec.TelemetryStore.Spec.Version) + }, + }, + } + + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + castingPath := filepath.Join(dir, "casting.yaml") + err := os.WriteFile(castingPath, []byte(tc.input), 0644) + require.NoError(t, err) + + cfg := New() + casting, err := cfg.GetV1Alpha1(context.Background(), castingPath) + require.NoError(t, err) + + inst, ok := casting.(*installation.Casting) + require.True(t, ok, "expected *installation.Casting, got %T", casting) + tc.assert(t, *inst) + }) + } +} + +func TestGetV1Alpha1Merge(t *testing.T) { + testCases := []struct { + name string + base installation.Casting + override installation.Casting + assert func(t *testing.T, casting installation.Casting) + }{ + { + name: "EmptyOverride", + base: *installation.Default(), + override: installation.Casting{}, + assert: func(t *testing.T, casting installation.Casting) { + assert.True(t, *casting.Spec.Signoz.Spec.Enabled) + assert.True(t, *casting.Spec.TelemetryStore.Spec.Enabled) + assert.True(t, *casting.Spec.MetaStore.Spec.Enabled) + assert.True(t, *casting.Spec.Ingester.Spec.Enabled) + }, + }, + { + name: "DisabledMoldingOverride", + base: *installation.Default(), + override: installation.Casting{ + Spec: installation.Spec{ + MetaStore: installation.MetaStore{ + Spec: v1alpha1.MoldingSpec{ + Enabled: domain.NewBoolPtr(false), + }, + }, + }, + }, + assert: func(t *testing.T, casting installation.Casting) { + assert.False(t, *casting.Spec.MetaStore.Spec.Enabled) + // Other moldings should remain enabled + assert.True(t, *casting.Spec.Signoz.Spec.Enabled) + assert.True(t, *casting.Spec.TelemetryStore.Spec.Enabled) + assert.True(t, *casting.Spec.Ingester.Spec.Enabled) + }, + }, + } + + t.Parallel() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + base := tc.base + override := tc.override + err := v1alpha1.Merge(&base, &override) + require.NoError(t, err) + tc.assert(t, base) + }) + } +} diff --git a/internal/domain/address.go b/internal/domain/address.go new file mode 100644 index 0000000..f677bab --- /dev/null +++ b/internal/domain/address.go @@ -0,0 +1,101 @@ +package domain + +import ( + "net/url" + "strconv" + "strings" + + "github.com/signoz/foundry/internal/errors" +) + +type Address struct { + scheme string + host string + port int +} + +// NewAddress requires a non-empty scheme and host, and a port in [0, 65535] where 0 means no port. +func NewAddress(scheme, host string, port int) (Address, error) { + if scheme == "" { + return Address{}, errors.Newf(errors.TypeInvalidInput, "failed to create address: scheme is empty") + } + + if host == "" { + return Address{}, errors.Newf(errors.TypeInvalidInput, "failed to create address: host is empty") + } + + if port < 0 || port > 65535 { + return Address{}, errors.Newf(errors.TypeInvalidInput, "failed to create address: port %d is out of range", port) + } + + return Address{scheme: scheme, host: host, port: port}, nil +} + +func MustNewAddress(scheme, host string, port int) Address { + address, err := NewAddress(scheme, host, port) + if err != nil { + panic(err) + } + + return address +} + +// ParseAddress accepts "scheme://host[:port]"; IPv6 literals must be bracketed +// (e.g. "tcp://[::1]:9000" or "tcp://[::1]"). +func ParseAddress(raw string) (Address, error) { + u, err := url.Parse(raw) + if err != nil { + return Address{}, errors.Wrapf(err, errors.TypeInvalidInput, "failed to create address from %q: contents are not a valid URL", raw) + } + + var port int + if portStr := u.Port(); portStr != "" { + port, err = strconv.Atoi(portStr) + if err != nil { + return Address{}, errors.Wrapf(err, errors.TypeInvalidInput, "failed to create address from %q: port is not a valid integer", raw) + } + } + + return NewAddress(u.Scheme, u.Hostname(), port) +} + +func ParseAddresses(raws []string) ([]Address, error) { + result := make([]Address, 0, len(raws)) + + for i, raw := range raws { + address, err := ParseAddress(raw) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInvalidInput, "addresses[%d]", i) + } + + result = append(result, address) + } + + return result, nil +} + +func (a Address) Scheme() string { + return a.scheme +} + +func (a Address) Host() string { + return a.host +} + +func (a Address) Port() int { + return a.port +} + +// String renders the address as "scheme://host[:port]", bracketing IPv6 hosts and omitting the port suffix when port is zero. +func (a Address) String() string { + host := a.host + if strings.Contains(host, ":") { + host = "[" + host + "]" + } + + if a.port == 0 { + return a.scheme + "://" + host + } + + return a.scheme + "://" + host + ":" + strconv.Itoa(a.port) +} diff --git a/internal/domain/address_test.go b/internal/domain/address_test.go new file mode 100644 index 0000000..eae796c --- /dev/null +++ b/internal/domain/address_test.go @@ -0,0 +1,85 @@ +package domain + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseAddress(t *testing.T) { + tests := []struct { + name string + raw string + pass bool + expectedScheme string + expectedHost string + expectedPort int + }{ + { + name: "Hostname_Valid", + raw: "tcp://query-service:9000", + pass: true, + expectedScheme: "tcp", + expectedHost: "query-service", + expectedPort: 9000, + }, + { + name: "IPv4_Valid", + raw: "postgres://127.0.0.1:5432", + pass: true, + expectedScheme: "postgres", + expectedHost: "127.0.0.1", + expectedPort: 5432, + }, + { + name: "IPv6_Valid", + raw: "tcp://[::1]:9000", + pass: true, + expectedScheme: "tcp", + expectedHost: "::1", + expectedPort: 9000, + }, + { + name: "Hostname_NoPort_Valid", + raw: "tcp://signoz", + pass: true, + expectedScheme: "tcp", + expectedHost: "signoz", + expectedPort: 0, + }, + { + name: "IPv6_NoPort_Valid", + raw: "tcp://[::1]", + pass: true, + expectedScheme: "tcp", + expectedHost: "::1", + expectedPort: 0, + }, + { + name: "MissingScheme_Invalid", + raw: "localhost:9000", + pass: false, + }, + { + name: "Malformed_Invalid", + raw: "://bad", + pass: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + address, err := ParseAddress(tt.raw) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.expectedScheme, address.Scheme()) + assert.Equal(t, tt.expectedHost, address.Host()) + assert.Equal(t, tt.expectedPort, address.Port()) + assert.Equal(t, tt.raw, address.String()) + }) + } +} diff --git a/internal/domain/blob_material.go b/internal/domain/blob_material.go new file mode 100644 index 0000000..d63c681 --- /dev/null +++ b/internal/domain/blob_material.go @@ -0,0 +1,23 @@ +package domain + +var _ Material = BlobMaterial{} + +type BlobMaterial struct { + contents []byte + path string +} + +func NewBlobMaterial(contents []byte, path string) BlobMaterial { + return BlobMaterial{ + contents: contents, + path: path, + } +} + +func (m BlobMaterial) Path() string { + return m.path +} + +func (m BlobMaterial) FmtContents() []byte { + return m.contents +} diff --git a/internal/domain/blob_material_test.go b/internal/domain/blob_material_test.go new file mode 100644 index 0000000..289a7c7 --- /dev/null +++ b/internal/domain/blob_material_test.go @@ -0,0 +1,32 @@ +package domain + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewBlobMaterial(t *testing.T) { + tests := []struct { + name string + contents []byte + path string + expectedFmtContents []byte + }{ + { + name: "Dockerfile", + contents: []byte("FROM alpine\nRUN echo '{not-json}'\n"), + path: "Dockerfile", + expectedFmtContents: []byte("FROM alpine\nRUN echo '{not-json}'\n"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + material := NewBlobMaterial(tt.contents, tt.path) + + assert.Equal(t, tt.path, material.Path()) + assert.Equal(t, tt.expectedFmtContents, material.FmtContents()) + }) + } +} diff --git a/internal/domain/distinct_id.go b/internal/domain/distinct_id.go new file mode 100644 index 0000000..317d43e --- /dev/null +++ b/internal/domain/distinct_id.go @@ -0,0 +1,44 @@ +package domain + +import ( + "github.com/denisbrodbeck/machineid" +) + +const unknownDistinctID = "unknown" + +// DistinctID is an anonymous, stable identifier for telemetry attribution. It +// is at most 32 characters and never reveals the source from which it was +// derived; values longer than 32 characters are truncated by NewDistinctID. +type DistinctID struct { + value string +} + +func NewDistinctID() (DistinctID, error) { + id, err := machineid.ProtectedID("foundryctl") + if err != nil { + return UnknownDistinctID(), nil + } + if len(id) > 32 { + id = id[:32] + } + + return DistinctID{value: id}, nil +} + +func MustNewDistinctID() DistinctID { + id, err := NewDistinctID() + if err != nil { + panic(err) + } + + return id +} + +// UnknownDistinctID is the sentinel used when no stable raw identifier could be derived. +func UnknownDistinctID() DistinctID { + return DistinctID{value: unknownDistinctID} +} + +func (d DistinctID) String() string { + return d.value +} diff --git a/internal/domain/doc.go b/internal/domain/doc.go new file mode 100644 index 0000000..2d58604 --- /dev/null +++ b/internal/domain/doc.go @@ -0,0 +1,16 @@ +// Package domain contains Foundry's shared domain primitives. +// +// This package is for value objects and pure helpers that are used across +// configuration loading, molding, casting, patching, and writing. Examples +// include structured materials, opaque blob materials, template wrappers, +// addresses, formats, pointer helpers, and serialization helpers. +// +// Keep this package free of orchestration and side effects. It must not decide +// which pipeline phase runs, select deployment implementations, execute tools, +// write files, read user configuration, or contain platform-specific rendering +// behavior. Put that logic in the package that owns the behavior instead. +// +// The only Foundry package this package may import is internal/errors. Do not +// import api/v1alpha1 or any internal orchestration, casting, molding, +// infrastructure, writer, tooler, config, or ledger package from here. +package domain diff --git a/internal/domain/event.go b/internal/domain/event.go new file mode 100644 index 0000000..28f06fc --- /dev/null +++ b/internal/domain/event.go @@ -0,0 +1,66 @@ +package domain + +import "github.com/signoz/foundry/internal/errors" + +const ( + eventOutcomeSucceeded = "succeeded" + eventOutcomeFailed = "failed" +) + +// Event names a foundryctl pipeline operation tracked by the ledger. An Event +// carries an optional past-tense outcome (succeeded or failed) so that ledger +// adapters see a finite, stable vocabulary like "forge succeeded" or +// "forge failed" rather than a single name disambiguated by a property. +type Event struct { + name string + outcome string +} + +var ( + EventGauge = Event{name: "gauge"} + EventForge = Event{name: "forge"} + EventCast = Event{name: "cast"} + EventCatalog = Event{name: "catalog"} +) + +var allEvents = []Event{EventGauge, EventForge, EventCast, EventCatalog} + +// NewEvent accepts only the names of declared base Event values. The returned +// Event has no outcome; use Succeeded or Failed to attach one. +func NewEvent(s string) (Event, error) { + for _, e := range allEvents { + if e.name == s { + return e, nil + } + } + + return Event{}, errors.Newf(errors.TypeInvalidInput, "failed to create event from %q: name is not a known event", s) +} + +func MustNewEvent(s string) Event { + e, err := NewEvent(s) + if err != nil { + panic(err) + } + + return e +} + +// Succeeded returns a copy of e with the success outcome attached. +func (e Event) Succeeded() Event { + return Event{name: e.name, outcome: eventOutcomeSucceeded} +} + +// Failed returns a copy of e with the failure outcome attached. +func (e Event) Failed() Event { + return Event{name: e.name, outcome: eventOutcomeFailed} +} + +// String renders the event as "<name>" or "<name> <outcome>". +func (e Event) String() string { + if e.outcome == "" { + return e.name + } + + return e.name + " " + e.outcome +} diff --git a/internal/types/file.go b/internal/domain/file.go similarity index 91% rename from internal/types/file.go rename to internal/domain/file.go index a0193d2..3b250a5 100644 --- a/internal/types/file.go +++ b/internal/domain/file.go @@ -1,4 +1,4 @@ -package types +package domain import "embed" diff --git a/internal/domain/ini_material.go b/internal/domain/ini_material.go new file mode 100644 index 0000000..aae4563 --- /dev/null +++ b/internal/domain/ini_material.go @@ -0,0 +1,157 @@ +package domain + +import ( + "bytes" + "encoding/json" + "fmt" + "slices" + + "github.com/signoz/foundry/internal/errors" + "gopkg.in/ini.v1" +) + +func init() { + ini.PrettyFormat = false +} + +var _ StructuredMaterial = INIMaterial{} + +type INIMaterial struct { + path string + contents []byte +} + +func NewINIMaterial(contents []byte, path string) (INIMaterial, error) { + cfg, err := ini.LoadSources(ini.LoadOptions{AllowShadows: true}, contents) + if err != nil { + return INIMaterial{}, errors.Wrapf(err, errors.TypeInvalidInput, "failed to create INI material for path %q: contents are not valid INI", path) + } + + data := make(map[string]map[string]any) + + for _, section := range cfg.Sections() { + if section.Name() == ini.DefaultSection && len(section.Keys()) == 0 { + continue + } + + sectionData := make(map[string]any) + for _, key := range section.Keys() { + vals := key.ValueWithShadows() + + if len(vals) > 1 { + sectionData[key.Name()] = vals + } else { + sectionData[key.Name()] = key.String() + } + } + data[section.Name()] = sectionData + } + + jsonContents, err := json.Marshal(data) + if err != nil { + return INIMaterial{}, errors.Wrapf(err, errors.TypeInternal, "failed to convert INI material to canonical JSON for path %q", path) + } + + return INIMaterial{ + path: path, + contents: jsonContents, + }, nil +} + +func MustNewINIMaterial(contents []byte, path string) INIMaterial { + material, err := NewINIMaterial(contents, path) + if err != nil { + panic(err) + } + + return material +} + +func (m INIMaterial) Path() string { + return m.path +} + +func (m INIMaterial) JSONContents() []byte { + return m.contents +} + +func (m INIMaterial) HasMultipleDocuments() bool { + return false +} + +func (m INIMaterial) FmtContents() []byte { + var data map[string]map[string]any + if err := json.Unmarshal(m.contents, &data); err != nil { + return nil + } + + cfg, err := ini.LoadSources(ini.LoadOptions{AllowShadows: true, PreserveSurroundedQuote: true}, []byte("")) + if err != nil { + return nil + } + + for _, sName := range getSortedKeys(data) { + section, _ := cfg.NewSection(sName) + + for _, kName := range getSortedKeys(data[sName]) { + if err := writeEntry(section, kName, data[sName][kName]); err != nil { + return nil + } + } + } + + var buf bytes.Buffer + if _, err := cfg.WriteTo(&buf); err != nil { + return nil + } + + return buf.Bytes() +} + +func (m INIMaterial) CloneWithJSONContents(contents []byte) StructuredMaterial { + return INIMaterial{ + contents: contents, + path: m.path, + } +} + +func (m INIMaterial) GetBytes(path string) ([]byte, error) { + return getBytes(m.contents, path) +} + +func (m INIMaterial) GetStringSlice(path string) ([]string, error) { + return getStringSlice(m.contents, path) +} + +func writeEntry(sec *ini.Section, key string, value any) error { + if vals, ok := value.([]any); ok { + for i, v := range vals { + strVal := fmt.Sprint(v) + if i == 0 { + if _, err := sec.NewKey(key, strVal); err != nil { + return err + } + } else { + if err := sec.Key(key).AddShadow(strVal); err != nil { + return err + } + } + } + return nil + } + + if _, err := sec.NewKey(key, fmt.Sprint(value)); err != nil { + return err + } + + return nil +} + +func getSortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + slices.Sort(keys) + return keys +} diff --git a/internal/domain/ini_material_test.go b/internal/domain/ini_material_test.go new file mode 100644 index 0000000..2144bb3 --- /dev/null +++ b/internal/domain/ini_material_test.go @@ -0,0 +1,90 @@ +package domain + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewINIMaterial(t *testing.T) { + tests := []struct { + name string + contents []byte + path string + pass bool + expectedJSON []byte + expectedFmtContents []byte + expectedHasMultipleDocuments bool + }{ + { + name: "OneSection_Valid", + contents: []byte("[Service]\nRestart=always\nEnvironment=SIGNOZ=1\nEnvironment=OTEL=1\n"), + path: "service.ini", + pass: true, + expectedJSON: []byte(`{"Service":{"Restart":"always","Environment":["SIGNOZ=1","OTEL=1"]}}`), + expectedFmtContents: []byte("[Service]\nEnvironment=SIGNOZ=1\nEnvironment=OTEL=1\nRestart=always\n"), + expectedHasMultipleDocuments: false, + }, + { + name: "TwoSections_Valid", + contents: []byte("[Service]\nRestart=always\nEnvironment=SIGNOZ=1\nEnvironment=OTEL=1\n[Database]\nHost=localhost\nPort=5432\n"), + path: "service.ini", + pass: true, + expectedJSON: []byte(`{"Service":{"Restart":"always","Environment":["SIGNOZ=1","OTEL=1"]},"Database":{"Host":"localhost","Port":"5432"}}`), + expectedFmtContents: []byte("[Database]\nHost=localhost\nPort=5432\n\n[Service]\nEnvironment=SIGNOZ=1\nEnvironment=OTEL=1\nRestart=always\n"), + expectedHasMultipleDocuments: false, + }, + { + name: "Invalid", + contents: []byte("[Service\nRestart=always\n"), + path: "service.ini", + pass: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + material, err := NewINIMaterial(tt.contents, tt.path) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.path, material.Path()) + assert.Equal(t, tt.expectedHasMultipleDocuments, material.HasMultipleDocuments()) + assert.JSONEq(t, string(tt.expectedJSON), string(material.JSONContents())) + assert.Equal(t, tt.expectedFmtContents, material.FmtContents()) + }) + } +} + +func TestINIMaterialGetBytes(t *testing.T) { + tests := []struct { + name string + material INIMaterial + path string + pass bool + expected []byte + }{ + { + name: "OneSection_Exists", + material: MustNewINIMaterial([]byte("[Service]\nRestart=always\nEnvironment=SIGNOZ=1\nEnvironment=OTEL=1\n"), "service.ini"), + path: "Service.Restart", + pass: true, + expected: []byte("always"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + output, err := tt.material.GetBytes(tt.path) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.Equal(t, tt.expected, output) + }) + } +} diff --git a/internal/domain/json_material.go b/internal/domain/json_material.go new file mode 100644 index 0000000..e9ff3f2 --- /dev/null +++ b/internal/domain/json_material.go @@ -0,0 +1,65 @@ +package domain + +import ( + "encoding/json" + + "github.com/signoz/foundry/internal/errors" +) + +var _ StructuredMaterial = JSONMaterial{} + +type JSONMaterial struct { + path string + contents []byte +} + +func NewJSONMaterial(contents []byte, path string) (JSONMaterial, error) { + if !json.Valid(contents) { + return JSONMaterial{}, errors.Newf(errors.TypeInvalidInput, "failed to create JSON material for path %q: contents are not valid JSON", path) + } + + return JSONMaterial{ + contents: contents, + path: path, + }, nil +} + +func MustNewJSONMaterial(contents []byte, path string) JSONMaterial { + material, err := NewJSONMaterial(contents, path) + if err != nil { + panic(err) + } + + return material +} + +func (m JSONMaterial) Path() string { + return m.path +} + +func (m JSONMaterial) JSONContents() []byte { + return m.contents +} + +func (m JSONMaterial) HasMultipleDocuments() bool { + return false +} + +func (m JSONMaterial) FmtContents() []byte { + return m.contents +} + +func (m JSONMaterial) CloneWithJSONContents(contents []byte) StructuredMaterial { + return JSONMaterial{ + contents: contents, + path: m.path, + } +} + +func (m JSONMaterial) GetBytes(path string) ([]byte, error) { + return getBytes(m.contents, path) +} + +func (m JSONMaterial) GetStringSlice(path string) ([]string, error) { + return getStringSlice(m.contents, path) +} diff --git a/internal/domain/json_material_test.go b/internal/domain/json_material_test.go new file mode 100644 index 0000000..d4806e5 --- /dev/null +++ b/internal/domain/json_material_test.go @@ -0,0 +1,81 @@ +package domain + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewJSONMaterial(t *testing.T) { + tests := []struct { + name string + contents []byte + path string + pass bool + expectedJSON []byte + expectedFmtContents []byte + expectedHasMultipleDocuments bool + }{ + { + name: "Valid", + contents: []byte(`{"service":{"names":["query-service","frontend"]}}`), + path: "service.json", + pass: true, + expectedJSON: []byte(`{"service":{"names":["query-service","frontend"]}}`), + expectedFmtContents: []byte(`{"service":{"names":["query-service","frontend"]}}`), + expectedHasMultipleDocuments: false, + }, + { + name: "Invalid", + contents: []byte(`{"service":`), + path: "service.json", + pass: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + material, err := NewJSONMaterial(tt.contents, tt.path) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.path, material.Path()) + assert.Equal(t, tt.expectedHasMultipleDocuments, material.HasMultipleDocuments()) + assert.JSONEq(t, string(tt.expectedJSON), string(material.JSONContents())) + assert.Equal(t, tt.expectedFmtContents, material.FmtContents()) + }) + } +} + +func TestJSONMaterialGetBytes(t *testing.T) { + tests := []struct { + name string + material JSONMaterial + path string + pass bool + expected []byte + }{ + { + name: "Exists", + material: MustNewJSONMaterial([]byte(`{"service":{"name":"query-service"}}`), "service.json"), + path: "service.name", + pass: true, + expected: []byte("query-service"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + output, err := tt.material.GetBytes(tt.path) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.Equal(t, tt.expected, output) + }) + } +} diff --git a/internal/domain/material.go b/internal/domain/material.go new file mode 100644 index 0000000..bf70e30 --- /dev/null +++ b/internal/domain/material.go @@ -0,0 +1,97 @@ +package domain + +import ( + "github.com/signoz/foundry/internal/errors" + "github.com/tidwall/gjson" +) + +var ( + FormatYAML = Format{s: "yaml", new: func(c []byte, p string) (Material, error) { return NewYAMLMaterial(c, p) }} + FormatJSON = Format{s: "json", new: func(c []byte, p string) (Material, error) { return NewJSONMaterial(c, p) }} + FormatINI = Format{s: "ini", new: func(c []byte, p string) (Material, error) { return NewINIMaterial(c, p) }} + FormatText = Format{s: "text", new: func(c []byte, p string) (Material, error) { return NewBlobMaterial(c, p), nil }} +) + +// Format identifies the syntax of a Material's contents and carries the +// constructor that wraps a rendered byte stream in the matching concrete +// Material type. To register a new format, add a single entry above with its +// New* constructor — Template.Render picks it up automatically. +type Format struct { + s string + new func(contents []byte, path string) (Material, error) +} + +func (f Format) String() string { + return f.s +} + +// NewMaterial wraps contents in the concrete Material type for this format. +func (f Format) NewMaterial(contents []byte, path string) (Material, error) { + if f.new == nil { + return nil, errors.Newf(errors.TypeUnsupported, "failed to create material for path %q: unsupported format %q", path, f.s) + } + return f.new(contents, path) +} + +// Material is a unit of output that Foundry produces. It carries the path it +// should be written to and the bytes to write there. +type Material interface { + Path() string + + // FmtContents returns the bytes in their human-readable, on-disk form. This + // is the form Foundry writes out, distinct from the canonical form used for + // traversal and patching. + FmtContents() []byte +} + +// StructuredMaterial is a Material whose contents are structured data with a +// navigable shape, supporting in-place reads and patches against a canonical +// representation. +type StructuredMaterial interface { + Material + + // JSONContents returns the canonical JSON form used for traversal and + // patching. JSON is the contract: callers (e.g. jsonpatch) operate on it + // directly. + JSONContents() []byte + + // HasMultipleDocuments reports whether the material groups multiple + // top-level documents under one path (currently only multi-document YAML). + // Callers use this to choose between scalar and array traversal of + // JSONContents. + HasMultipleDocuments() bool + + CloneWithJSONContents(contents []byte) StructuredMaterial + + // GetBytes returns the value at the given path as bytes. The path uses + // gjson dotted-key syntax (e.g. "service.name", "service.names.0"), not + // JSON Pointer. + GetBytes(path string) ([]byte, error) + + // GetStringSlice returns the slice at the given path as strings. See + // GetBytes for path syntax. + GetStringSlice(path string) ([]string, error) +} + +func getBytes(contents []byte, path string) ([]byte, error) { + result := gjson.GetBytes(contents, path) + if !result.Exists() { + return nil, errors.Newf(errors.TypeNotFound, "path %q does not exist", path) + } + + return []byte(result.String()), nil +} + +func getStringSlice(contents []byte, path string) ([]string, error) { + result := gjson.GetBytes(contents, path) + if !result.Exists() { + return nil, errors.Newf(errors.TypeNotFound, "path %q does not exist", path) + } + + var items []string + for _, item := range result.Array() { + items = append(items, item.String()) + } + + return items, nil +} diff --git a/internal/domain/pointer.go b/internal/domain/pointer.go new file mode 100644 index 0000000..f8d87d0 --- /dev/null +++ b/internal/domain/pointer.go @@ -0,0 +1,9 @@ +package domain + +func NewIntPtr(i int) *int { + return &i +} + +func NewBoolPtr(i bool) *bool { + return &i +} diff --git a/internal/domain/properties.go b/internal/domain/properties.go new file mode 100644 index 0000000..79c03ff --- /dev/null +++ b/internal/domain/properties.go @@ -0,0 +1,63 @@ +package domain + +import ( + "maps" + + "github.com/signoz/foundry/internal/errors" +) + +const ( + propertyKeySuccess = "success" + propertyKeyError = "error" + propertyKeyErrorType = "error_type" + propertyKeyErrorCause = "error_cause" +) + +// Properties is a string-keyed bag of telemetry values with a fixed shape for +// the success/error envelope (see WithSuccess and WithError). Set, WithSuccess, +// and WithError mutate the underlying map and return the receiver for chaining. +type Properties struct { + values map[string]any +} + +func NewProperties() Properties { + return Properties{values: make(map[string]any)} +} + +func (p Properties) Set(key string, value any) Properties { + p.values[key] = value + return p +} + +// WithSuccess records that the tracked operation succeeded. +func (p Properties) WithSuccess() Properties { + p.values[propertyKeySuccess] = true + return p +} + +// WithError records the typed kind, the outer message, and the next link's +// own Message as the cause — using the link's Message (not the chained +// Error()) gives stable grouping keys for analytics. +func (p Properties) WithError(err error) Properties { + p.values[propertyKeySuccess] = false + + e := errors.ExceptionOf(err) + if e == nil { + return p + } + + p.values[propertyKeyErrorType] = e.Type + p.values[propertyKeyError] = e.Message + if e.Cause != nil { + p.values[propertyKeyErrorCause] = e.Cause.Message + } + + return p +} + +// Map returns a copy of the underlying values, safe for callers to mutate. +func (p Properties) Map() map[string]any { + out := make(map[string]any, len(p.values)) + maps.Copy(out, p.values) + return out +} diff --git a/internal/domain/template.go b/internal/domain/template.go new file mode 100644 index 0000000..ecc5b1a --- /dev/null +++ b/internal/domain/template.go @@ -0,0 +1,156 @@ +package domain + +import ( + "bytes" + "embed" + "io" + "path/filepath" + "text/template" + + "github.com/Masterminds/sprig/v3" + "github.com/signoz/foundry/internal/errors" + "sigs.k8s.io/yaml" +) + +// Template is a parsed text/template paired with the on-disk path it was loaded +// from and the Format its rendered output is expected to take. Format drives +// Render's choice of concrete Material type. +type Template struct { + name string + path string + format Format + tmpl *template.Template +} + +func NewTemplateFromFS(fs embed.FS, path string, format Format) (*Template, error) { + name := filepath.Base(path) + tmpl, err := template.New(name).Funcs(templateFuncMap()).ParseFS(fs, path) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInvalidInput, "failed to create template from %q: contents are not a valid template", path) + } + + return &Template{name: name, path: path, format: format, tmpl: tmpl}, nil +} + +func NewTemplate(name string, contents []byte, format Format) (*Template, error) { + tmpl, err := template.New(name).Funcs(templateFuncMap()).Parse(string(contents)) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInvalidInput, "failed to create template from %q: contents are not a valid template", name) + } + + return &Template{name: name, format: format, tmpl: tmpl}, nil +} + +func MustNewTemplateFromFS(fs embed.FS, path string, format Format) *Template { + tmpl, err := NewTemplateFromFS(fs, path, format) + if err != nil { + panic(err) + } + + return tmpl +} + +func MustNewTemplate(name string, contents []byte, format Format) *Template { + tmpl, err := NewTemplate(name, contents, format) + if err != nil { + panic(err) + } + + return tmpl +} + +// Execute renders the template into w. Each call clones the underlying +// *template.Template so concurrent calls don't share parse state. +func (t *Template) Execute(w io.Writer, data any) error { + newtmpl, err := t.tmpl.Clone() + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to execute template %q", t.name) + } + + if err := newtmpl.ExecuteTemplate(w, t.name, data); err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to execute template %q", t.name) + } + + return nil +} + +// Render executes the template against data and wraps the output in the +// concrete Material type indicated by the template's Format. +func (t *Template) Render(data any, path string) (Material, error) { + buf := bytes.NewBuffer(nil) + if err := t.Execute(buf, data); err != nil { + return nil, err + } + + m, err := t.format.NewMaterial(buf.Bytes(), path) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to render template %q", t.name) + } + return m, nil +} + +func (t *Template) Name() string { + return t.name +} + +func (t *Template) Path() string { + return t.path +} + +func (t *Template) Format() Format { + return t.format +} + +// templateFuncMap layers the project's helpers on top of sprig: +// - derefInt / derefIntDefault: nil-safe int pointer access for spec fields +// that distinguish "unset" from zero. +// - toYaml / fromYaml: round-trip a value through YAML inside templates. +// - flattenKeys: flatten a nested map into "/"-joined leaf keys (used to +// project hierarchical config into flat env-var-style maps). +func templateFuncMap() template.FuncMap { + fm := template.FuncMap(sprig.FuncMap()) + fm["derefInt"] = func(p *int) int { + if p == nil { + return 0 + } + return *p + } + fm["derefIntDefault"] = func(p *int, defaultVal int) int { + if p == nil { + return defaultVal + } + return *p + } + fm["toYaml"] = func(v any) (string, error) { + if v == nil { + return "", nil + } + b, err := yaml.Marshal(v) + return string(b), err + } + fm["fromYaml"] = func(s string) (map[string]any, error) { + var m map[string]any + err := yaml.Unmarshal([]byte(s), &m) + return m, err + } + fm["flattenKeys"] = func(m map[string]any) map[string]any { + result := make(map[string]any) + flattenMapKeys("", m, result) + return result + } + return fm +} + +func flattenMapKeys(prefix string, m map[string]any, result map[string]any) { + for k, v := range m { + key := k + if prefix != "" { + key = prefix + "/" + k + } + if nested, ok := v.(map[string]any); ok { + flattenMapKeys(key, nested, result) + } else { + result[key] = v + } + } +} diff --git a/internal/domain/template_test.go b/internal/domain/template_test.go new file mode 100644 index 0000000..8bddc22 --- /dev/null +++ b/internal/domain/template_test.go @@ -0,0 +1,88 @@ +package domain + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTemplateRender(t *testing.T) { + tests := []struct { + name string + contents []byte + format Format + path string + data any + pass bool + expectedFmt []byte + }{ + { + name: "YAML_Valid", + contents: []byte("greeting: hello {{ .Name }}\n"), + format: FormatYAML, + path: "out.yaml", + data: map[string]string{"Name": "world"}, + pass: true, + expectedFmt: []byte("greeting: hello world\n"), + }, + { + name: "JSON_Valid", + contents: []byte(`{"greeting":"hello {{ .Name }}"}`), + format: FormatJSON, + path: "out.json", + data: map[string]string{"Name": "world"}, + pass: true, + expectedFmt: []byte(`{"greeting":"hello world"}`), + }, + { + name: "INI_Valid", + contents: []byte("[greeting]\nname = {{ .Name }}\n"), + format: FormatINI, + path: "out.ini", + data: map[string]string{"Name": "world"}, + pass: true, + expectedFmt: []byte("[greeting]\nname=world\n"), + }, + { + name: "Text_Valid", + contents: []byte("hello {{ .Name }}"), + format: FormatText, + path: "out.txt", + data: map[string]string{"Name": "world"}, + pass: true, + expectedFmt: []byte("hello world"), + }, + { + name: "UnsupportedFormat_Invalid", + contents: []byte("hello"), + format: Format{s: "bogus"}, + path: "out", + data: nil, + pass: false, + }, + { + name: "JSON_InvalidOutput", + contents: []byte(`{"greeting": hello {{ .Name }}}`), + format: FormatJSON, + path: "out.json", + data: map[string]string{"Name": "world"}, + pass: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpl := MustNewTemplate(tt.name, tt.contents, tt.format) + + material, err := tmpl.Render(tt.data, tt.path) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.path, material.Path()) + assert.Equal(t, tt.expectedFmt, material.FmtContents()) + }) + } +} diff --git a/internal/domain/yaml.go b/internal/domain/yaml.go new file mode 100644 index 0000000..36f4c96 --- /dev/null +++ b/internal/domain/yaml.go @@ -0,0 +1,78 @@ +package domain + +import ( + "github.com/signoz/foundry/internal/errors" + kyaml "sigs.k8s.io/yaml" +) + +func UnmarshalYAML(data []byte, v any) error { + return kyaml.Unmarshal(data, v) +} + +func MustUnmarshalYAML(data []byte, v any) error { + return kyaml.Unmarshal(data, v) +} + +func MarshalYAML(v any) ([]byte, error) { + yaml, err := kyaml.Marshal(v) + if err != nil { + return nil, err + } + + return yaml, nil +} + +func MustMarshalYAML(v any) []byte { + yaml, err := MarshalYAML(v) + if err != nil { + panic(err) + } + + return yaml +} + +// MergeYAML takes a base YAML string and an override YAML string, +// and returns a new YAML string with deep merge — override keys win +// at every level, base-only keys are preserved. +func MergeYAML(base, override string) (string, error) { + var baseMap map[string]any + if err := kyaml.Unmarshal([]byte(base), &baseMap); err != nil { + return "", errors.Wrapf(err, errors.TypeInternal, "failed to unmarshal base yaml") + } + + var overrideMap map[string]any + if err := kyaml.Unmarshal([]byte(override), &overrideMap); err != nil { + return "", errors.Wrapf(err, errors.TypeInternal, "failed to unmarshal override yaml") + } + + deepMerge(baseMap, overrideMap) + + merged, err := kyaml.Marshal(baseMap) + if err != nil { + return "", errors.Wrapf(err, errors.TypeInternal, "failed to marshal merged yaml") + } + + return string(merged), nil +} + +// deepMerge recursively merges override into base. +// For matching keys: if both values are maps, merge recursively. +// Otherwise, override wins. +func deepMerge(base, override map[string]any) { + for k, overrideVal := range override { + baseVal, exists := base[k] + if !exists { + base[k] = overrideVal + continue + } + + baseMap, baseIsMap := baseVal.(map[string]any) + overrideMap, overrideIsMap := overrideVal.(map[string]any) + + if baseIsMap && overrideIsMap { + deepMerge(baseMap, overrideMap) + } else { + base[k] = overrideVal + } + } +} diff --git a/internal/domain/yaml_material.go b/internal/domain/yaml_material.go new file mode 100644 index 0000000..d8f1ae8 --- /dev/null +++ b/internal/domain/yaml_material.go @@ -0,0 +1,135 @@ +package domain + +import ( + "bytes" + "encoding/json" + + "github.com/signoz/foundry/internal/errors" + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/yaml" + kyaml "sigs.k8s.io/yaml" +) + +var _ StructuredMaterial = YAMLMaterial{} + +type YAMLMaterial struct { + path string + contents []byte + hasMultipleDocuments bool +} + +func NewYAMLMaterial(contents []byte, path string) (YAMLMaterial, error) { + reader := kio.ByteReader{Reader: bytes.NewReader(contents), OmitReaderAnnotations: true} + + nodes, err := reader.Read() + if err != nil { + return YAMLMaterial{}, errors.Wrapf(err, errors.TypeInvalidInput, "failed to create YAML material for path %q: contents are not valid YAML", path) + } + + var jsonContents []byte + if len(nodes) == 1 { + jsonContents, err = nodes[0].MarshalJSON() + if err != nil { + return YAMLMaterial{}, errors.Wrapf(err, errors.TypeInternal, "failed to convert YAML material to canonical JSON for path %q", path) + } + + return YAMLMaterial{ + contents: jsonContents, + path: path, + hasMultipleDocuments: false, + }, nil + } + + var docs []json.RawMessage + for _, node := range nodes { + jsonContents, err := node.MarshalJSON() + if err != nil { + return YAMLMaterial{}, errors.Wrapf(err, errors.TypeInternal, "failed to convert YAML material to canonical JSON for path %q", path) + } + + docs = append(docs, jsonContents) + } + + jsonContents, err = json.Marshal(docs) + if err != nil { + return YAMLMaterial{}, errors.Wrapf(err, errors.TypeInternal, "failed to convert multi-document YAML material to canonical JSON for path %q", path) + } + + return YAMLMaterial{ + contents: jsonContents, + path: path, + hasMultipleDocuments: len(nodes) > 1, + }, nil +} + +func MustNewYAMLMaterial(contents []byte, path string) YAMLMaterial { + material, err := NewYAMLMaterial(contents, path) + if err != nil { + panic(err) + } + + return material +} + +func (m YAMLMaterial) Path() string { + return m.path +} + +func (m YAMLMaterial) JSONContents() []byte { + return m.contents +} + +func (m YAMLMaterial) HasMultipleDocuments() bool { + return m.hasMultipleDocuments +} + +func (m YAMLMaterial) FmtContents() []byte { + if !m.HasMultipleDocuments() { + node, err := kyaml.JSONToYAML(m.contents) + if err != nil { + return nil + } + + return node + } + + var docs []json.RawMessage + if err := json.Unmarshal(m.contents, &docs); err != nil { + return nil + } + + var nodes []*yaml.RNode + for _, doc := range docs { + node, err := yaml.ConvertJSONToYamlNode(string(doc)) + if err != nil { + return nil + } + + nodes = append(nodes, node) + } + + var buf bytes.Buffer + writer := kio.ByteWriter{Writer: &buf, KeepReaderAnnotations: true} + + if err := writer.Write(nodes); err != nil { + return nil + } + + return buf.Bytes() +} + +func (m YAMLMaterial) CloneWithJSONContents(contents []byte) StructuredMaterial { + return YAMLMaterial{ + contents: contents, + path: m.path, + hasMultipleDocuments: m.hasMultipleDocuments, + } +} + +func (m YAMLMaterial) GetBytes(path string) ([]byte, error) { + return getBytes(m.contents, path) +} + +func (m YAMLMaterial) GetStringSlice(path string) ([]string, error) { + return getStringSlice(m.contents, path) +} diff --git a/internal/domain/yaml_material_test.go b/internal/domain/yaml_material_test.go new file mode 100644 index 0000000..b5902bb --- /dev/null +++ b/internal/domain/yaml_material_test.go @@ -0,0 +1,90 @@ +package domain + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewYAMLMaterial(t *testing.T) { + tests := []struct { + name string + contents []byte + path string + pass bool + expectedJSON []byte + expectedFmtContents []byte + expectedHasMultipleDocuments bool + }{ + { + name: "SingleDocument_Valid", + contents: []byte("service:\n names:\n - query-service\n - frontend\n"), + path: "service.yaml", + pass: true, + expectedJSON: []byte(`{"service":{"names":["query-service","frontend"]}}`), + expectedFmtContents: []byte("service:\n names:\n - query-service\n - frontend\n"), + expectedHasMultipleDocuments: false, + }, + { + name: "MultiDocument_Valid", + contents: []byte("---\nname: one\n---\nname: two\n"), + path: "service.yaml", + pass: true, + expectedJSON: []byte(`[{"name":"one"},{"name":"two"}]`), + expectedFmtContents: []byte("name: one\n---\nname: two\n"), + expectedHasMultipleDocuments: true, + }, + { + name: "Invalid", + contents: []byte("service: [unterminated\n"), + path: "service.yaml", + pass: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + material, err := NewYAMLMaterial(tt.contents, tt.path) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.path, material.Path()) + assert.Equal(t, tt.expectedHasMultipleDocuments, material.HasMultipleDocuments()) + assert.JSONEq(t, string(tt.expectedJSON), string(material.JSONContents())) + assert.Equal(t, tt.expectedFmtContents, material.FmtContents()) + }) + } +} + +func TestYAMLMaterialGetBytes(t *testing.T) { + tests := []struct { + name string + material YAMLMaterial + path string + pass bool + expected []byte + }{ + { + name: "SingleDocument_Exists", + material: MustNewYAMLMaterial([]byte("service:\n name: query-service\n"), "service.yaml"), + path: "service.name", + pass: true, + expected: []byte("query-service"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + output, err := tt.material.GetBytes(tt.path) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.Equal(t, tt.expected, output) + }) + } +} diff --git a/internal/errors/error.go b/internal/errors/error.go index d5b4411..d70eeb6 100644 --- a/internal/errors/error.go +++ b/internal/errors/error.go @@ -1,10 +1,77 @@ package errors -import "log/slog" +import ( + goerrors "errors" + "fmt" + "log/slog" +) -func LogAttr(err error) slog.Attr { - return slog.Attr{ - Key: "err", - Value: slog.StringValue(err.Error()), +type base struct { + // t denotes the custom type of the error. + t typ + + // info contains the error message + info string + + // cause is the actual error which is being wrapped with a stacktrace and message information. + cause error + + // s contains the stacktrace captured at error creation time. + stacktrace fmt.Stringer +} + +func (b *base) Error() string { + if b.cause != nil { + return fmt.Sprintf("%s: %s", b.info, b.cause.Error()) + } + + return b.info +} + +func (b *base) Unwrap() error { + return b.cause +} + +func (b *base) WithStacktrace(stacktrace string) *base { + b.stacktrace = rawStacktrace(stacktrace) + return b +} + +func (b *base) Stacktrace() string { + return b.stacktrace.String() +} + +func Newf(t typ, info string, args ...any) *base { + return &base{ + t: t, + info: fmt.Sprintf(info, args...), + cause: nil, + stacktrace: newStackTrace(), } } + +func Wrapf(cause error, t typ, format string, args ...any) error { + return &base{ + t: t, + info: fmt.Sprintf(format, args...), + cause: cause, + stacktrace: newStackTrace(), + } +} + +func ExitCode(err error) int { + if err == nil { + return 0 + } + + var b *base + if goerrors.As(err, &b) { + return b.t.ExitCode() + } + + return 1 +} + +func LogAttr(err error) slog.Attr { + return slog.GroupAttrs("exception", exceptionAttrs(ExceptionOf(err))...) +} diff --git a/internal/errors/error_test.go b/internal/errors/error_test.go new file mode 100644 index 0000000..36ab414 --- /dev/null +++ b/internal/errors/error_test.go @@ -0,0 +1,40 @@ +package errors + +import ( + "fmt" + "testing" +) + +func TestExitCode(t *testing.T) { + tests := []struct { + name string + err error + expected int + }{ + {name: "Nil_Zero", err: nil, expected: 0}, + {name: "Untyped_One", err: fmt.Errorf("plain error"), expected: 1}, + {name: "InvalidInput_Two", err: Newf(TypeInvalidInput, "bad input"), expected: 2}, + {name: "NotFound_Three", err: Newf(TypeNotFound, "missing"), expected: 3}, + {name: "Unsupported_Four", err: Newf(TypeUnsupported, "no support"), expected: 4}, + {name: "Internal_Five", err: Newf(TypeInternal, "internal"), expected: 5}, + {name: "Fatal_Six", err: Newf(TypeFatal, "fatal"), expected: 6}, + { + name: "WrappedWithFmtErrorf_WalksChain", + err: fmt.Errorf("outer: %w", Newf(TypeNotFound, "inner")), + expected: 3, + }, + { + name: "WrappedWithFoundryWrapf_PreservesOuterType", + err: Wrapf(Newf(TypeNotFound, "inner"), TypeUnsupported, "outer"), + expected: 4, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ExitCode(tt.err); got != tt.expected { + t.Errorf("ExitCode() = %d, want %d", got, tt.expected) + } + }) + } +} diff --git a/internal/errors/exception.go b/internal/errors/exception.go new file mode 100644 index 0000000..bccf1a7 --- /dev/null +++ b/internal/errors/exception.go @@ -0,0 +1,81 @@ +package errors + +import ( + "encoding/json" + "log/slog" +) + +type Exception struct { + Type string `json:"type,omitempty"` + Message string `json:"message"` + Cause *Exception `json:"cause,omitempty"` + Action string `json:"action,omitempty"` + Stacktrace string `json:"stacktrace,omitempty"` +} + +type Envelope struct { + Exception *Exception +} + +func (e Envelope) MarshalJSON() ([]byte, error) { + return json.MarshalIndent(map[string]*Exception{"exception": e.Exception}, "", " ") +} + +// The walk terminates at the first non-*base link, which emits Message alone +// — stdlib wrappers format their full subtree in Error(), so re-walking them +// would duplicate that text. Stacktrace emits on TypeFatal links only; every +// *base captures one at construction time but emitting them all is noise. +func ExceptionOf(err error) *Exception { + if err == nil { + return nil + } + + b, ok := err.(*base) + if !ok { + return &Exception{Message: err.Error()} + } + + e := &Exception{ + Type: b.t.String(), + Message: b.info, + Action: b.t.action, + Cause: ExceptionOf(b.cause), + } + if b.t == TypeFatal && b.stacktrace != nil { + if st := b.stacktrace.String(); st != "" { + e.Stacktrace = st + } + } + + return e +} + +func EnvelopeOf(err error) Envelope { + return Envelope{Exception: ExceptionOf(err)} +} + +func exceptionAttrs(e *Exception) []slog.Attr { + if e == nil { + return nil + } + + var attrs []slog.Attr + if e.Type != "" { + attrs = append(attrs, slog.String("type", e.Type)) + } + + attrs = append(attrs, slog.String("message", e.Message)) + if e.Cause != nil { + attrs = append(attrs, slog.GroupAttrs("cause", exceptionAttrs(e.Cause)...)) + } + + if e.Action != "" { + attrs = append(attrs, slog.String("action", e.Action)) + } + + if e.Stacktrace != "" { + attrs = append(attrs, slog.String("stacktrace", e.Stacktrace)) + } + + return attrs +} diff --git a/internal/errors/stacktrace.go b/internal/errors/stacktrace.go new file mode 100644 index 0000000..9de53e5 --- /dev/null +++ b/internal/errors/stacktrace.go @@ -0,0 +1,45 @@ +package errors + +import ( + "fmt" + "runtime" + "strings" +) + +// stacktrace holds a snapshot of program counters. +type stacktrace []uintptr + +// newStackTrace captures a stack trace, skipping 3 frames to record the +// snapshot at the origin of the error: +// 1. runtime.Callers +// 2. newStackTrace +// 3. the constructor (New, Newf, Wrapf, Wrap) +// +// Inspired by https://github.com/thanos-io/thanos/blob/main/pkg/errors/stacktrace.go +func newStackTrace() stacktrace { + const depth = 16 + pc := make([]uintptr, depth) + n := runtime.Callers(3, pc) + return stacktrace(pc[:n:n]) +} + +// String formats the stacktrace as function/file/line pairs. +func (s stacktrace) String() string { + var buf strings.Builder + frames := runtime.CallersFrames(s) + for { + frame, more := frames.Next() + fmt.Fprintf(&buf, "%s\n\t%s:%d\n", frame.Function, frame.File, frame.Line) + if !more { + break + } + } + return buf.String() +} + +// rawStacktrace holds a pre-formatted stacktrace string. +type rawStacktrace string + +func (r rawStacktrace) String() string { + return string(r) +} diff --git a/internal/errors/type.go b/internal/errors/type.go new file mode 100644 index 0000000..03afb44 --- /dev/null +++ b/internal/errors/type.go @@ -0,0 +1,41 @@ +package errors + +// Exit codes use a compact 1-6 scheme rather than BSD sysexits.h. sysexits +// would force TypeInternal and TypeFatal to share EX_SOFTWARE (70), and the +// distinction matters: TypeFatal marks a recovered panic that should page, +// TypeInternal marks an expected-but-failed path. The custom scheme keeps +// those orthogonal and stays easy to remember in shell scripts. +// +// The action string is the remediation hint surfaced to users (via the slog +// "exception" payload and the --format=json error envelope). Co-locating it +// with the type means adding a new error class is a one-place change. +var ( + TypeInvalidInput = typ{"invalid-input", 2, ""} + TypeNotFound = typ{"not-found", 3, ""} + TypeUnsupported = typ{"unsupported", 4, "Please check the documentation for supported features or raise an issue at https://github.com/signoz/foundry/issues for feature requests."} + TypeInternal = typ{"internal", 5, ""} + TypeFatal = typ{"fatal", 6, "Please raise an issue at https://github.com/signoz/foundry/issues with the error message and stacktrace."} +) + +// Defines custom error types, the process exit code they map to, and the +// remediation hint shown to users. +type typ struct { + s string + code int + action string +} + +func (t typ) String() string { + return t.s +} + +// ExitCode returns the process exit code associated with this error type. +func (t typ) ExitCode() int { + return t.code +} + +// Action returns the remediation hint for this error type, or an empty string +// if no hint applies. +func (t typ) Action() string { + return t.action +} diff --git a/internal/foundry/cast.go b/internal/foundry/cast.go index 3b36ddc..51e1feb 100644 --- a/internal/foundry/cast.go +++ b/internal/foundry/cast.go @@ -2,25 +2,14 @@ package foundry import ( "context" - "log/slog" "github.com/hanzoai/o11y-foundry/api/v1alpha1" ) -func (foundry *Foundry) Cast(ctx context.Context, config v1alpha1.Casting, poursPath string) error { - foundry.Logger.InfoContext(ctx, "starting cast pipeline", slog.String("casting.metadata.name", config.Metadata.Name)) - - // Get the casting for the deployment mode - casting, err := foundry.Registry.Casting(config.Spec.Deployment) +func (foundry *Foundry) Cast(ctx context.Context, machinery v1alpha1.Machinery, poursPath string) error { + p, err := foundry.newPlanner(ctx, machinery) if err != nil { return err } - - err = casting.Cast(ctx, config, poursPath) - if err != nil { - foundry.Logger.ErrorContext(ctx, err.Error()) - return err - } - - return nil + return p.Cast(ctx, poursPath) } diff --git a/internal/foundry/forge.go b/internal/foundry/forge.go index e2a8ba6..997a833 100644 --- a/internal/foundry/forge.go +++ b/internal/foundry/forge.go @@ -2,8 +2,8 @@ package foundry import ( "context" - "fmt" "log/slog" + "path/filepath" "github.com/hanzoai/o11y-foundry/api/v1alpha1" foundryerrors "github.com/hanzoai/o11y-foundry/internal/errors" @@ -11,63 +11,77 @@ import ( "github.com/hanzoai/o11y-foundry/internal/writer" ) -func (foundry *Foundry) Forge(ctx context.Context, config v1alpha1.Casting, path string, poursWriterOpts *writer.Options) error { - foundry.Logger.InfoContext(ctx, "starting forge pipeline", slog.String("casting.metadata.name", config.Metadata.Name)) - - casting, err := foundry.Registry.Casting(config.Spec.Deployment) +func (foundry *Foundry) Forge(ctx context.Context, machinery v1alpha1.Machinery, path string, poursWriterOpts *writer.Options) error { + p, err := foundry.newPlanner(ctx, machinery) if err != nil { - foundry.Logger.ErrorContext(ctx, "casting not found", slog.String("casting.spec.deployment.mode", config.Spec.Deployment.Mode)) return err } - foundry.Logger.InfoContext(ctx, "enriching moldings with casting specific information", slog.String("casting.metadata.name", config.Metadata.Name)) - moldingEnricher, err := casting.Enricher(ctx, &config) - if err != nil { - foundry.Logger.ErrorContext(ctx, "failed to get molding enricher", slog.String("casting.metadata.name", config.Metadata.Name), foundryerrors.LogAttr(err)) - return fmt.Errorf("failed to get molding enricher: %w", err) - } - - foundry.Logger.InfoContext(ctx, "enriching configuration with casting specific information", slog.String("casting.metadata.name", config.Metadata.Name)) - for _, moldingKind := range molding.MoldingsInOrder() { - err = moldingEnricher.EnrichStatus(ctx, moldingKind, &config) - if err != nil { - return fmt.Errorf("failed to enrich configuration with casting specific information: %w", err) + for _, kind := range p.MoldingKinds() { + if err := p.EnrichStatus(ctx, kind); err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to enrich molding %s", kind) } } - // Molding the configuration - for _, molding := range molding.MoldingsInOrder() { - foundry.Logger.InfoContext(ctx, "molding configuration for kind", slog.String("molding.kind", molding.String())) - err = foundry.Moldings[molding].MoldV1Alpha1(ctx, &config) - if err != nil { - foundry.Logger.ErrorContext(ctx, "failed to mold configuration", slog.String("molding.kind", molding.String()), foundryerrors.LogAttr(err)) + for _, kind := range p.MoldingKinds() { + foundry.Logger.InfoContext(ctx, "molding configuration for kind", slog.String("molding.kind", kind.String())) + if err := p.Mold(ctx, kind); err != nil { return err } } - // merging status into spec - foundry.Logger.InfoContext(ctx, "merging status into spec", slog.String("casting.metadata.name", config.Metadata.Name)) - if err := v1alpha1.MergeCastingSpecAndStatus(&config); err != nil { - foundry.Logger.ErrorContext(ctx, "failed to merge status into spec", slog.String("casting.metadata.name", config.Metadata.Name), foundryerrors.LogAttr(err)) + if err := p.MergeStatusIntoSpec(); err != nil { return err } - // Forging the configuration - foundry.Logger.InfoContext(ctx, "forging configuration with the merged spec and generating materials", slog.String("casting.metadata.name", config.Metadata.Name)) - materials, err := casting.Forge(ctx, config, poursWriterOpts.TargetDirectory) + materials, err := p.Forge(ctx, poursWriterOpts.TargetDirectory) if err != nil { return err } - // writing the merged config to the config file - foundry.Logger.InfoContext(ctx, "writing lock file", slog.String("casting.metadata.name", config.Metadata.Name)) + for _, pe := range p.Patches() { + patcher, ok := foundry.Patchers[pe.PatchType()] + if !ok { + return foundryerrors.Newf(foundryerrors.TypeUnsupported, "unknown patch type %q", pe.PatchType()) + } + foundry.Logger.InfoContext(ctx, "applying patch", slog.String("patch.type", pe.PatchType()), slog.String("patch.target", pe.Target)) + materials, err = patcher.Apply(ctx, materials, pe) + if err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to apply patch for target %q", pe.Target) + } + } - err = foundry.Config.CreateV1Alpha1Lock(ctx, config, path) - if err != nil { + // Generate infrastructure-as-code manifests if enabled, before writing the lock file + // so that the generated file contents are captured in the lock's infrastructure.status. + // Gated to installation.Casting + var infraMaterials []domain.Material + if config, ok := machinery.(*installation.Casting); ok && config.Spec.Infrastructure.Enabled { + spec := &config.Spec + foundry.Logger.InfoContext(ctx, "generating infrastructure manifests", + slog.String("casting.metadata.name", config.Metadata.Name), + slog.String("deployment.platform", spec.Deployment.Platform.String())) + + infraMaterials, err = foundry.InfrastructureGenerator.Generate(ctx, *config) + if err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to generate infrastructure manifests") + } + + // Populate infrastructure status with generated file contents keyed by filename. + if len(infraMaterials) > 0 { + spec.Infrastructure.Status = make(map[string]string, len(infraMaterials)) + for _, m := range infraMaterials { + spec.Infrastructure.Status[filepath.Base(m.Path())] = string(m.FmtContents()) + } + } + } + + // writing the merged config (including infrastructure status) to the lock file + foundry.Logger.InfoContext(ctx, "writing lock file") + if err := foundry.Config.CreateV1Alpha1Lock(ctx, p.Machinery(), path); err != nil { return err } - if len(materials) == 0 { + if len(materials) == 0 && len(infraMaterials) == 0 { foundry.Logger.WarnContext(ctx, "casting did not generate any materials for writing") return nil } @@ -77,12 +91,16 @@ func (foundry *Foundry) Forge(ctx context.Context, config v1alpha1.Casting, path return err } - // Writing the materials - foundry.Logger.InfoContext(ctx, "writing materials", slog.String("casting.metadata.name", config.Metadata.Name)) - err = poursWriter.WriteMany(ctx, materials...) - if err != nil { - return err + if len(infraMaterials) > 0 { + foundry.Logger.InfoContext(ctx, "writing infrastructure materials", slog.Int("count", len(infraMaterials))) + if err := poursWriter.WriteMany(ctx, infraMaterials...); err != nil { + return err + } } + foundry.Logger.InfoContext(ctx, "writing materials") + if err := poursWriter.WriteMany(ctx, materials...); err != nil { + return err + } return nil } diff --git a/internal/foundry/foundry.go b/internal/foundry/foundry.go index 7a79ec6..106ad24 100644 --- a/internal/foundry/foundry.go +++ b/internal/foundry/foundry.go @@ -1,6 +1,7 @@ package foundry import ( + "context" "log/slog" "github.com/hanzoai/o11y-foundry/api/v1alpha1" @@ -14,28 +15,26 @@ import ( "github.com/hanzoai/o11y-foundry/internal/molding/telemetrystoremolding" ) +type plannerCtor func(ctx context.Context, m v1alpha1.Machinery, logger *slog.Logger) (planner.Planner, error) + type Foundry struct { // Config for loading the casting configuration. Config config.Config + // Patchers for applying patches to generated materials, keyed by patch type. + Patchers map[string]patch.Patch + // Logger for logging. Logger *slog.Logger - // Registry for the different deployments. - Registry *Registry + // Planners for the different casting kinds. + Planners map[v1alpha1.Kind]plannerCtor - // Moldings for the different molding kinds. - Moldings map[v1alpha1.MoldingKind]molding.Molding + // InfrastructureGenerator for generating infrastructure-as-code manifests. + InfrastructureGenerator infrastructure.Generator } func New(logger *slog.Logger) (*Foundry, error) { - yamlConfig := yamlconfig.New() - - registry, err := NewRegistry(logger) - if err != nil { - return nil, err - } - return &Foundry{ Config: yamlConfig, Logger: logger, @@ -47,5 +46,23 @@ func New(logger *slog.Logger) (*Foundry, error) { v1alpha1.MoldingKindO11y: o11ymolding.New(logger), v1alpha1.MoldingKindIngester: ingestermolding.New(logger), }, + Logger: logger, + Planners: map[v1alpha1.Kind]plannerCtor{ + v1alpha1.KindInstallation: func(ctx context.Context, m v1alpha1.Machinery, logger *slog.Logger) (planner.Planner, error) { + return installationcasting.NewPlanner(ctx, m.(*installation.Casting), logger) + }, + v1alpha1.KindCollectionAgent: func(ctx context.Context, m v1alpha1.Machinery, logger *slog.Logger) (planner.Planner, error) { + return collectionagentcasting.NewPlanner(ctx, m.(*collectionagent.Casting), logger) + }, + }, + InfrastructureGenerator: terraformgenerator.New(logger), }, nil } + +func (foundry *Foundry) newPlanner(ctx context.Context, m v1alpha1.Machinery) (planner.Planner, error) { + ctor, ok := foundry.Planners[m.Kind()] + if !ok { + return nil, foundryerrors.Newf(foundryerrors.TypeUnsupported, "unsupported casting kind %q", m.Kind()) + } + return ctor(ctx, m, foundry.Logger) +} diff --git a/internal/foundry/gauge.go b/internal/foundry/gauge.go index cc0af83..89b5718 100644 --- a/internal/foundry/gauge.go +++ b/internal/foundry/gauge.go @@ -2,7 +2,6 @@ package foundry import ( "context" - "fmt" "log/slog" "strings" @@ -10,30 +9,23 @@ import ( foundryerrors "github.com/hanzoai/o11y-foundry/internal/errors" ) -func (foundry *Foundry) Gauge(ctx context.Context, config v1alpha1.Casting) error { - foundry.Logger.InfoContext(ctx, "starting gauge pipeline", slog.String("casting.metadata.name", config.Metadata.Name)) - - toolers, err := foundry.Registry.Toolers(config.Spec.Deployment) +func (foundry *Foundry) Gauge(ctx context.Context, machinery v1alpha1.Machinery) error { + p, err := foundry.newPlanner(ctx, machinery) if err != nil { return err } unavailableTools := []string{} - - for _, tooler := range toolers { - err := tooler.Gauge(ctx) - if err != nil { + for _, tooler := range p.Toolers() { + if err := tooler.Gauge(ctx); err != nil { foundry.Logger.ErrorContext(ctx, "tool is not available or cannot be detected properly", slog.String("tool.name", tooler.Name()), foundryerrors.LogAttr(err)) unavailableTools = append(unavailableTools, tooler.Name()) continue } - foundry.Logger.InfoContext(ctx, "tool is available", slog.String("tool.name", tooler.Name())) } - if len(unavailableTools) > 0 { - return fmt.Errorf("tools are not available, please install them and try again: %s", strings.Join(unavailableTools, ", ")) + return foundryerrors.Newf(foundryerrors.TypeNotFound, "tools are not available, please install them and try again: %s", strings.Join(unavailableTools, ", ")) } - return nil } diff --git a/internal/foundry/registry.go b/internal/foundry/registry.go deleted file mode 100644 index 98f0989..0000000 --- a/internal/foundry/registry.go +++ /dev/null @@ -1,82 +0,0 @@ -package foundry - -import ( - "fmt" - "log/slog" - - "github.com/hanzoai/o11y-foundry/api/v1alpha1" - "github.com/hanzoai/o11y-foundry/internal/casting" - "github.com/hanzoai/o11y-foundry/internal/casting/dockercomposecasting" - "github.com/hanzoai/o11y-foundry/internal/casting/rendercasting" - "github.com/hanzoai/o11y-foundry/internal/casting/systemdcasting" - "github.com/hanzoai/o11y-foundry/internal/tooler" - "github.com/hanzoai/o11y-foundry/internal/tooler/clickhousekeepertooler" - "github.com/hanzoai/o11y-foundry/internal/tooler/clickhousetooler" - "github.com/hanzoai/o11y-foundry/internal/tooler/dockercomposetooler" - "github.com/hanzoai/o11y-foundry/internal/tooler/dockertooler" - "github.com/hanzoai/o11y-foundry/internal/tooler/postgrestooler" - "github.com/hanzoai/o11y-foundry/internal/tooler/systemdtooler" -) - -// Defines a single casting item in the registry. -type CastingItem struct { - // The particular casting implementation. - Casting casting.Casting - - // The toolers for the particular casting. - Toolers []tooler.Tooler -} - -type Registry struct { - // Castings for the different deployments. - castings map[v1alpha1.TypeDeployment]CastingItem -} - -func NewRegistry(logger *slog.Logger) (*Registry, error) { - return &Registry{ - castings: map[v1alpha1.TypeDeployment]CastingItem{ - { - Mode: "docker", - Flavor: "compose", - }: { - Casting: dockercomposecasting.New(logger), - Toolers: []tooler.Tooler{dockertooler.New(), dockercomposetooler.New()}, - }, - { - Mode: "systemd", - Flavor: "binary", - }: { - Casting: systemdcasting.New(logger), - Toolers: []tooler.Tooler{systemdtooler.New(), clickhousekeepertooler.New(), clickhousetooler.New(), postgrestooler.New()}, - }, - { - Platform: "render", - Flavor: "blueprint", - }: { - Casting: rendercasting.New(logger), - }, - }, - }, nil -} - -func (registry *Registry) CastingItems() map[v1alpha1.TypeDeployment]CastingItem { - return registry.castings -} - -func (registry *Registry) Casting(deployment v1alpha1.TypeDeployment) (casting.Casting, error) { - item, ok := registry.castings[deployment] - if !ok { - return nil, fmt.Errorf("deployment '%+v' is not supported, raise an issue at https://github.com/hanzoai/o11y-foundry/issues to request support for this deployment", deployment) - } - - return item.Casting, nil -} - -func (registry *Registry) Toolers(deployment v1alpha1.TypeDeployment) ([]tooler.Tooler, error) { - item, ok := registry.castings[deployment] - if !ok { - return nil, fmt.Errorf("deployment '%+v' is not supported, raise an issue at https://github.com/hanzoai/o11y-foundry/issues to request support for this deployment", deployment) - } - - return item.Toolers, nil -} diff --git a/internal/infrastructure/compute_type.go b/internal/infrastructure/compute_type.go new file mode 100644 index 0000000..37e77f7 --- /dev/null +++ b/internal/infrastructure/compute_type.go @@ -0,0 +1,91 @@ +package infrastructure + +import ( + "encoding/json" + "errors" + "fmt" + + "go.yaml.in/yaml/v3" +) + +var _ yaml.Marshaler = (*ComputeType)(nil) +var _ yaml.Unmarshaler = (*ComputeType)(nil) +var _ json.Marshaler = (*ComputeType)(nil) +var _ json.Unmarshaler = (*ComputeType)(nil) +var _ fmt.Stringer = (*ComputeType)(nil) + +var ( + // AWS compute types. + ComputeTypeEC2 ComputeType = ComputeType{s: "ec2"} + ComputeTypeEKS ComputeType = ComputeType{s: "eks"} + // GCP compute types. + ComputeTypeGCE ComputeType = ComputeType{s: "gce"} + ComputeTypeGKE ComputeType = ComputeType{s: "gke"} + // Azure compute types. + ComputeTypeVM ComputeType = ComputeType{s: "vm"} + ComputeTypeAKS ComputeType = ComputeType{s: "aks"} +) + +// ComputeType identifies the compute resource type for a given cloud provider. +// It is an internal type resolved from the provider + deployment combination — +// users do not set this directly. +type ComputeType struct { + s string +} + +func (c ComputeType) String() string { + return c.s +} + +func (c ComputeType) IsZero() bool { + return c.s == "" +} + +func ComputeTypes() []ComputeType { + return []ComputeType{ + ComputeTypeEC2, + ComputeTypeEKS, + ComputeTypeGCE, + ComputeTypeGKE, + ComputeTypeVM, + ComputeTypeAKS, + } +} + +func (c ComputeType) MarshalJSON() ([]byte, error) { + return json.Marshal(c.String()) +} + +func (c *ComputeType) UnmarshalJSON(text []byte) error { + var str string + if err := json.Unmarshal(text, &str); err != nil { + return err + } + return c.UnmarshalText([]byte(str)) +} + +func (c *ComputeType) UnmarshalText(text []byte) error { + for _, available := range ComputeTypes() { + if available.String() == string(text) { + *c = available + return nil + } + } + if len(text) == 0 { + *c = ComputeType{s: ""} + return nil + } + return errors.New("invalid infrastructure compute type: " + string(text)) +} + +func (c ComputeType) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + +func (c *ComputeType) UnmarshalYAML(node *yaml.Node) error { + return c.UnmarshalText([]byte(node.Value)) +} + +func (c ComputeType) MarshalYAML() (any, error) { + return c.String(), nil +} diff --git a/internal/infrastructure/generator.go b/internal/infrastructure/generator.go new file mode 100644 index 0000000..9e7983d --- /dev/null +++ b/internal/infrastructure/generator.go @@ -0,0 +1,20 @@ +package infrastructure + +import ( + "context" + + "github.com/signoz/foundry/api/v1alpha1/installation" + "github.com/signoz/foundry/internal/domain" +) + +// Generator is the interface for infrastructure-as-code generators. +// Implementations produce IaC manifests (e.g., Terraform, Pulumi) from a casting configuration +// and can validate the generated output using the underlying tool. +type Generator interface { + // Generate produces IaC materials from the casting configuration. + Generate(ctx context.Context, config installation.Casting) ([]domain.Material, error) + + // Validate runs the IaC tool's built-in validation (e.g., terraform validate) + // against the manifests written to poursPath. + Validate(ctx context.Context, poursPath string) error +} diff --git a/internal/infrastructure/resolve.go b/internal/infrastructure/resolve.go new file mode 100644 index 0000000..d8324cc --- /dev/null +++ b/internal/infrastructure/resolve.go @@ -0,0 +1,61 @@ +package infrastructure + +import ( + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/internal/errors" +) + +// ResolveProvider normalizes a deployment platform to the cloud platform that +// hosts it. Only the cloud platforms (aws, gcp, azure) and ECS resolve; +// managed platforms (render, coolify, railway) have no IaC backing. +func ResolveProvider(platform v1alpha1.Platform) (v1alpha1.Platform, error) { + switch platform { + case v1alpha1.PlatformAWS, v1alpha1.PlatformECS: + return v1alpha1.PlatformAWS, nil + case v1alpha1.PlatformGCP: + return v1alpha1.PlatformGCP, nil + case v1alpha1.PlatformAzure: + return v1alpha1.PlatformAzure, nil + case v1alpha1.Platform{}: + return v1alpha1.Platform{}, errors.Newf(errors.TypeInvalidInput, "no platform specified in deployment.platform: infrastructure generation requires aws, gcp, or azure") + default: + return v1alpha1.Platform{}, errors.Newf(errors.TypeUnsupported, "unsupported platform for infrastructure generation: %q (must be aws, gcp, azure, or ecs)", platform) + } +} + +// ResolveComputeType derives the appropriate ComputeType from a cloud platform +// and deployment configuration. Users do not specify the compute type directly +// — foundry resolves it automatically using this matrix: +// +// AWS + kubernetes (any flavor) → EKS +// AWS + anything else → EC2 +// GCP + kubernetes (any flavor) → GKE +// GCP + anything else → GCE +// Azure + kubernetes (any flavor) → AKS +// Azure + anything else → VM +func ResolveComputeType(provider v1alpha1.Platform, deployment v1alpha1.TypeDeployment) (ComputeType, error) { + isKubernetes := deployment.Mode == v1alpha1.ModeKubernetes + + switch provider { + case v1alpha1.PlatformAWS: + if isKubernetes { + return ComputeTypeEKS, nil + } + return ComputeTypeEC2, nil + + case v1alpha1.PlatformGCP: + if isKubernetes { + return ComputeTypeGKE, nil + } + return ComputeTypeGCE, nil + + case v1alpha1.PlatformAzure: + if isKubernetes { + return ComputeTypeAKS, nil + } + return ComputeTypeVM, nil + + default: + return ComputeType{}, errors.Newf(errors.TypeUnsupported, "unsupported infrastructure platform: %s", provider) + } +} diff --git a/internal/infrastructure/terraform/embed.go b/internal/infrastructure/terraform/embed.go new file mode 100644 index 0000000..914ab8d --- /dev/null +++ b/internal/infrastructure/terraform/embed.go @@ -0,0 +1,57 @@ +package terraform + +import ( + "embed" + + "github.com/signoz/foundry/internal/domain" +) + +//go:embed templates/*.gotmpl templates/aws/ec2/*.gotmpl templates/aws/eks/*.gotmpl templates/gcp/gce/*.gotmpl templates/gcp/gke/*.gotmpl templates/azure/vm/*.gotmpl templates/azure/aks/*.gotmpl +var templates embed.FS + +// Common templates. +var ( + providersTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/providers.tf.json.gotmpl", domain.FormatJSON) +) + +// AWS EC2 templates. +var ( + awsEC2MainTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/aws/ec2/main.tf.json.gotmpl", domain.FormatJSON) + awsEC2VariablesTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/aws/ec2/variables.tf.json.gotmpl", domain.FormatJSON) + awsEC2OutputsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/aws/ec2/outputs.tf.json.gotmpl", domain.FormatJSON) +) + +// AWS EKS templates. +var ( + awsEKSMainTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/aws/eks/main.tf.json.gotmpl", domain.FormatJSON) + awsEKSVariablesTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/aws/eks/variables.tf.json.gotmpl", domain.FormatJSON) + awsEKSOutputsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/aws/eks/outputs.tf.json.gotmpl", domain.FormatJSON) +) + +// GCP GCE templates. +var ( + gcpGCEMainTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/gcp/gce/main.tf.json.gotmpl", domain.FormatJSON) + gcpGCEVariablesTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/gcp/gce/variables.tf.json.gotmpl", domain.FormatJSON) + gcpGCEOutputsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/gcp/gce/outputs.tf.json.gotmpl", domain.FormatJSON) +) + +// GCP GKE templates. +var ( + gcpGKEMainTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/gcp/gke/main.tf.json.gotmpl", domain.FormatJSON) + gcpGKEVariablesTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/gcp/gke/variables.tf.json.gotmpl", domain.FormatJSON) + gcpGKEOutputsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/gcp/gke/outputs.tf.json.gotmpl", domain.FormatJSON) +) + +// Azure VM templates. +var ( + azureVMMainTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/azure/vm/main.tf.json.gotmpl", domain.FormatJSON) + azureVMVariablesTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/azure/vm/variables.tf.json.gotmpl", domain.FormatJSON) + azureVMOutputsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/azure/vm/outputs.tf.json.gotmpl", domain.FormatJSON) +) + +// Azure AKS templates. +var ( + azureAKSMainTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/azure/aks/main.tf.json.gotmpl", domain.FormatJSON) + azureAKSVariablesTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/azure/aks/variables.tf.json.gotmpl", domain.FormatJSON) + azureAKSOutputsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/azure/aks/outputs.tf.json.gotmpl", domain.FormatJSON) +) diff --git a/internal/infrastructure/terraform/generator.go b/internal/infrastructure/terraform/generator.go new file mode 100644 index 0000000..5b7a7be --- /dev/null +++ b/internal/infrastructure/terraform/generator.go @@ -0,0 +1,130 @@ +package terraform + +import ( + "context" + "log/slog" + "os/exec" + "path/filepath" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/installation" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/infrastructure" +) + +var _ infrastructure.Generator = (*Generator)(nil) + +const infrastructureDir = "infrastructure" + +// Generator generates Terraform manifests for infrastructure deployment. +type Generator struct { + logger *slog.Logger +} + +type templateData struct { + installation.Casting + Provider v1alpha1.Platform + ComputeType infrastructure.ComputeType +} + +// New creates a new Terraform Generator. +func New(logger *slog.Logger) *Generator { + return &Generator{ + logger: logger, + } +} + +// Generate creates Terraform manifests based on the casting configuration. +// The compute type is resolved automatically from the provider and deployment mode. +func (g *Generator) Generate(ctx context.Context, config installation.Casting) ([]domain.Material, error) { + if !config.Spec.Infrastructure.Enabled { + return nil, nil + } + + provider, err := infrastructure.ResolveProvider(config.Spec.Deployment.Platform) + if err != nil { + return nil, err + } + computeType, err := infrastructure.ResolveComputeType(provider, config.Spec.Deployment) + if err != nil { + return nil, err + } + + g.logger.InfoContext(ctx, "generating terraform manifests", + slog.String("provider", provider.String()), + slog.String("computeType", computeType.String()), + ) + + data := templateData{ + Casting: config, + Provider: provider, + ComputeType: computeType, + } + + mainTemplate, varsTemplate, outputsTemplate, err := g.templatesFor(provider, computeType) + if err != nil { + return nil, err + } + + materials := make([]domain.Material, 0, 4) + for _, item := range []struct { + tmpl *domain.Template + path string + }{ + {mainTemplate, "main.tf.json"}, + {varsTemplate, "variables.tf.json"}, + {providersTFTemplate, "providers.tf.json"}, + {outputsTemplate, "outputs.tf.json"}, + } { + m, err := item.tmpl.Render(data, filepath.Join(infrastructureDir, item.path)) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to render %s", item.path) + } + materials = append(materials, m) + } + + return materials, nil +} + +// Validate runs `terraform validate` against the manifests in poursPath/infrastructure. +func (g *Generator) Validate(ctx context.Context, poursPath string) error { + infraDir := filepath.Join(poursPath, infrastructureDir) + g.logger.InfoContext(ctx, "validating terraform manifests", slog.String("path", infraDir)) + + cmd := exec.CommandContext(ctx, "terraform", "validate") + cmd.Dir = infraDir + out, err := cmd.CombinedOutput() + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "terraform validate failed\n%s", out) + } + return nil +} + +// templatesFor returns the provider+compute-type specific templates. +func (g *Generator) templatesFor(provider v1alpha1.Platform, computeType infrastructure.ComputeType) (main, vars, outputs *domain.Template, err error) { + switch provider { + case v1alpha1.PlatformAWS: + switch computeType { + case infrastructure.ComputeTypeEC2: + return awsEC2MainTFTemplate, awsEC2VariablesTFTemplate, awsEC2OutputsTFTemplate, nil + case infrastructure.ComputeTypeEKS: + return awsEKSMainTFTemplate, awsEKSVariablesTFTemplate, awsEKSOutputsTFTemplate, nil + } + case v1alpha1.PlatformGCP: + switch computeType { + case infrastructure.ComputeTypeGCE: + return gcpGCEMainTFTemplate, gcpGCEVariablesTFTemplate, gcpGCEOutputsTFTemplate, nil + case infrastructure.ComputeTypeGKE: + return gcpGKEMainTFTemplate, gcpGKEVariablesTFTemplate, gcpGKEOutputsTFTemplate, nil + } + case v1alpha1.PlatformAzure: + switch computeType { + case infrastructure.ComputeTypeVM: + return azureVMMainTFTemplate, azureVMVariablesTFTemplate, azureVMOutputsTFTemplate, nil + case infrastructure.ComputeTypeAKS: + return azureAKSMainTFTemplate, azureAKSVariablesTFTemplate, azureAKSOutputsTFTemplate, nil + } + } + return nil, nil, nil, errors.Newf(errors.TypeUnsupported, "unsupported provider %q / compute type %q combination", provider, computeType) +} diff --git a/internal/infrastructure/terraform/templates/aws/ec2/main.tf.json.gotmpl b/internal/infrastructure/terraform/templates/aws/ec2/main.tf.json.gotmpl new file mode 100644 index 0000000..3ede88d --- /dev/null +++ b/internal/infrastructure/terraform/templates/aws/ec2/main.tf.json.gotmpl @@ -0,0 +1,456 @@ +{ + "locals": { + "name": "{{ .Metadata.Name }}", + "common_tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "data": { + "aws_availability_zones": { + "available": { + "state": "available" + } + }, + "aws_ami": { + "ubuntu": { + "most_recent": true, + "owners": ["099720109477"], + "filter": [ + { + "name": "name", + "values": ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"] + }, + { + "name": "virtualization-type", + "values": ["hvm"] + } + ] + } + } + }, + "resource": { + "aws_vpc": { + "main": { + "cidr_block": "${var.vpc_cidr}", + "enable_dns_hostnames": true, + "enable_dns_support": true, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-vpc" + } + } + }, + "aws_subnet": { + "private": { + "count": "${var.az_count}", + "vpc_id": "${aws_vpc.main.id}", + "cidr_block": "${cidrsubnet(var.vpc_cidr, 8, count.index)}", + "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-private-${count.index}" + } + }, + "public": { + "count": "${var.az_count}", + "vpc_id": "${aws_vpc.main.id}", + "cidr_block": "${cidrsubnet(var.vpc_cidr, 8, count.index + var.az_count)}", + "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", + "map_public_ip_on_launch": true, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-public-${count.index}" + } + } + }, + "aws_internet_gateway": { + "main": { + "vpc_id": "${aws_vpc.main.id}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-igw" + } + } + }, + "aws_eip": { + "nat": { + "count": "${var.az_count}", + "domain": "vpc", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-nat-eip-${count.index}" + } + } + }, + "aws_nat_gateway": { + "main": { + "count": "${var.az_count}", + "allocation_id": "${aws_eip.nat[count.index].id}", + "subnet_id": "${aws_subnet.public[count.index].id}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-nat-${count.index}" + }, + "depends_on": ["${aws_internet_gateway.main}"] + } + }, + "aws_route_table": { + "public": { + "vpc_id": "${aws_vpc.main.id}", + "route": [ + { + "cidr_block": "0.0.0.0/0", + "gateway_id": "${aws_internet_gateway.main.id}" + } + ], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-public-rt" + } + }, + "private": { + "count": "${var.az_count}", + "vpc_id": "${aws_vpc.main.id}", + "route": [ + { + "cidr_block": "0.0.0.0/0", + "nat_gateway_id": "${aws_nat_gateway.main[count.index].id}" + } + ], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-private-rt-${count.index}" + } + } + }, + "aws_route_table_association": { + "public": { + "count": "${var.az_count}", + "subnet_id": "${aws_subnet.public[count.index].id}", + "route_table_id": "${aws_route_table.public.id}" + }, + "private": { + "count": "${var.az_count}", + "subnet_id": "${aws_subnet.private[count.index].id}", + "route_table_id": "${aws_route_table.private[count.index].id}" + } + }, + "aws_security_group": { + "telemetrykeeper": { + "name": "${local.name}-telemetrykeeper-sg", + "description": "Security group for TelemetryKeeper (ClickHouse Keeper)", + "vpc_id": "${aws_vpc.main.id}", + "ingress": [ + { + "from_port": 9181, + "to_port": 9181, + "protocol": "tcp", + "cidr_blocks": ["${var.vpc_cidr}"], + "description": "ClickHouse Keeper client port" + }, + { + "from_port": 9234, + "to_port": 9234, + "protocol": "tcp", + "cidr_blocks": ["${var.vpc_cidr}"], + "description": "ClickHouse Keeper raft port" + }, + { + "from_port": 22, + "to_port": 22, + "protocol": "tcp", + "cidr_blocks": ["${var.vpc_cidr}"], + "description": "SSH access from VPC" + } + ], + "egress": [ + { + "from_port": 0, + "to_port": 0, + "protocol": "-1", + "cidr_blocks": ["0.0.0.0/0"] + } + ], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-telemetrykeeper-sg" + } + }, + "telemetrystore": { + "name": "${local.name}-telemetrystore-sg", + "description": "Security group for TelemetryStore (ClickHouse)", + "vpc_id": "${aws_vpc.main.id}", + "ingress": [ + { + "from_port": 9000, + "to_port": 9000, + "protocol": "tcp", + "cidr_blocks": ["${var.vpc_cidr}"], + "description": "ClickHouse native port" + }, + { + "from_port": 8123, + "to_port": 8123, + "protocol": "tcp", + "cidr_blocks": ["${var.vpc_cidr}"], + "description": "ClickHouse HTTP port" + }, + { + "from_port": 22, + "to_port": 22, + "protocol": "tcp", + "cidr_blocks": ["${var.vpc_cidr}"], + "description": "SSH access from VPC" + } + ], + "egress": [ + { + "from_port": 0, + "to_port": 0, + "protocol": "-1", + "cidr_blocks": ["0.0.0.0/0"] + } + ], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-telemetrystore-sg" + } + }, + "metastore": { + "name": "${local.name}-metastore-sg", + "description": "Security group for MetaStore (PostgreSQL)", + "vpc_id": "${aws_vpc.main.id}", + "ingress": [ + { + "from_port": 5432, + "to_port": 5432, + "protocol": "tcp", + "cidr_blocks": ["${var.vpc_cidr}"], + "description": "PostgreSQL port" + }, + { + "from_port": 22, + "to_port": 22, + "protocol": "tcp", + "cidr_blocks": ["${var.vpc_cidr}"], + "description": "SSH access from VPC" + } + ], + "egress": [ + { + "from_port": 0, + "to_port": 0, + "protocol": "-1", + "cidr_blocks": ["0.0.0.0/0"] + } + ], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-metastore-sg" + } + }, + "ingester": { + "name": "${local.name}-ingester-sg", + "description": "Security group for Ingester (OpenTelemetry Collector)", + "vpc_id": "${aws_vpc.main.id}", + "ingress": [ + { + "from_port": 4317, + "to_port": 4317, + "protocol": "tcp", + "cidr_blocks": ["0.0.0.0/0"], + "description": "OTLP gRPC port" + }, + { + "from_port": 4318, + "to_port": 4318, + "protocol": "tcp", + "cidr_blocks": ["0.0.0.0/0"], + "description": "OTLP HTTP port" + }, + { + "from_port": 22, + "to_port": 22, + "protocol": "tcp", + "cidr_blocks": ["${var.vpc_cidr}"], + "description": "SSH access from VPC" + } + ], + "egress": [ + { + "from_port": 0, + "to_port": 0, + "protocol": "-1", + "cidr_blocks": ["0.0.0.0/0"] + } + ], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-ingester-sg" + } + }, + "signoz": { + "name": "${local.name}-signoz-sg", + "description": "Security group for SigNoz", + "vpc_id": "${aws_vpc.main.id}", + "ingress": [ + { + "from_port": 8080, + "to_port": 8080, + "protocol": "tcp", + "cidr_blocks": ["0.0.0.0/0"], + "description": "SigNoz UI port" + }, + { + "from_port": 3301, + "to_port": 3301, + "protocol": "tcp", + "cidr_blocks": ["${var.vpc_cidr}"], + "description": "SigNoz API port" + }, + { + "from_port": 22, + "to_port": 22, + "protocol": "tcp", + "cidr_blocks": ["${var.vpc_cidr}"], + "description": "SSH access from VPC" + } + ], + "egress": [ + { + "from_port": 0, + "to_port": 0, + "protocol": "-1", + "cidr_blocks": ["0.0.0.0/0"] + } + ], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-signoz-sg" + } + } + }, + "aws_key_pair": { + "main": { + "count": "${var.ssh_public_key != \"\" ? 1 : 0}", + "key_name": "${local.name}-key", + "public_key": "${var.ssh_public_key}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + } + }, + "aws_instance": { + "telemetrykeeper": { + "count": {{ if .Spec.TelemetryKeeper.Spec.Enabled }}{{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "ami": "${data.aws_ami.ubuntu.id}", + "instance_type": "${var.telemetrykeeper_instance_type}", + "subnet_id": "${aws_subnet.private[count.index % var.az_count].id}", + "vpc_security_group_ids": ["${aws_security_group.telemetrykeeper.id}"], + "key_name": "${var.ssh_public_key != \"\" ? aws_key_pair.main[0].key_name : null}", + "root_block_device": [{ + "volume_size": "${var.telemetrykeeper_volume_size}", + "volume_type": "gp3", + "encrypted": true + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-telemetrykeeper-${count.index}", + "Role": "telemetrykeeper" + } + }, + "telemetrystore": { + "count": {{ if .Spec.TelemetryStore.Spec.Enabled }}{{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "ami": "${data.aws_ami.ubuntu.id}", + "instance_type": "${var.telemetrystore_instance_type}", + "subnet_id": "${aws_subnet.private[count.index % var.az_count].id}", + "vpc_security_group_ids": ["${aws_security_group.telemetrystore.id}"], + "key_name": "${var.ssh_public_key != \"\" ? aws_key_pair.main[0].key_name : null}", + "root_block_device": [{ + "volume_size": "${var.telemetrystore_volume_size}", + "volume_type": "gp3", + "encrypted": true + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-telemetrystore-${count.index}", + "Role": "telemetrystore" + } + }, + "metastore": { + "count": {{ if .Spec.MetaStore.Spec.Enabled }}{{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "ami": "${data.aws_ami.ubuntu.id}", + "instance_type": "${var.metastore_instance_type}", + "subnet_id": "${aws_subnet.private[count.index % var.az_count].id}", + "vpc_security_group_ids": ["${aws_security_group.metastore.id}"], + "key_name": "${var.ssh_public_key != \"\" ? aws_key_pair.main[0].key_name : null}", + "root_block_device": [{ + "volume_size": "${var.metastore_volume_size}", + "volume_type": "gp3", + "encrypted": true + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-metastore-${count.index}", + "Role": "metastore" + } + }, + "ingester": { + "count": {{ if .Spec.Ingester.Spec.Enabled }}{{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "ami": "${data.aws_ami.ubuntu.id}", + "instance_type": "${var.ingester_instance_type}", + "subnet_id": "${aws_subnet.public[count.index % var.az_count].id}", + "vpc_security_group_ids": ["${aws_security_group.ingester.id}"], + "key_name": "${var.ssh_public_key != \"\" ? aws_key_pair.main[0].key_name : null}", + "root_block_device": [{ + "volume_size": "${var.ingester_volume_size}", + "volume_type": "gp3", + "encrypted": true + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-ingester-${count.index}", + "Role": "ingester" + } + }, + "signoz": { + "count": {{ if .Spec.Signoz.Spec.Enabled }}{{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "ami": "${data.aws_ami.ubuntu.id}", + "instance_type": "${var.signoz_instance_type}", + "subnet_id": "${aws_subnet.public[count.index % var.az_count].id}", + "vpc_security_group_ids": ["${aws_security_group.signoz.id}"], + "key_name": "${var.ssh_public_key != \"\" ? aws_key_pair.main[0].key_name : null}", + "root_block_device": [{ + "volume_size": "${var.signoz_volume_size}", + "volume_type": "gp3", + "encrypted": true + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-signoz-${count.index}", + "Role": "signoz" + } + } + } + } +} diff --git a/internal/infrastructure/terraform/templates/aws/ec2/outputs.tf.json.gotmpl b/internal/infrastructure/terraform/templates/aws/ec2/outputs.tf.json.gotmpl new file mode 100644 index 0000000..38d7d0b --- /dev/null +++ b/internal/infrastructure/terraform/templates/aws/ec2/outputs.tf.json.gotmpl @@ -0,0 +1,64 @@ +{ + "output": { + "vpc_id": { + "description": "ID of the VPC", + "value": "${aws_vpc.main.id}" + }, + "private_subnet_ids": { + "description": "IDs of the private subnets", + "value": "${aws_subnet.private[*].id}" + }, + "public_subnet_ids": { + "description": "IDs of the public subnets", + "value": "${aws_subnet.public[*].id}" + }, + "telemetrykeeper_instance_ids": { + "description": "IDs of the TelemetryKeeper EC2 instances", + "value": "${aws_instance.telemetrykeeper[*].id}" + }, + "telemetrykeeper_private_ips": { + "description": "Private IP addresses of the TelemetryKeeper EC2 instances", + "value": "${aws_instance.telemetrykeeper[*].private_ip}" + }, + "telemetrystore_instance_ids": { + "description": "IDs of the TelemetryStore EC2 instances", + "value": "${aws_instance.telemetrystore[*].id}" + }, + "telemetrystore_private_ips": { + "description": "Private IP addresses of the TelemetryStore EC2 instances", + "value": "${aws_instance.telemetrystore[*].private_ip}" + }, + "metastore_instance_ids": { + "description": "IDs of the MetaStore EC2 instances", + "value": "${aws_instance.metastore[*].id}" + }, + "metastore_private_ips": { + "description": "Private IP addresses of the MetaStore EC2 instances", + "value": "${aws_instance.metastore[*].private_ip}" + }, + "ingester_instance_ids": { + "description": "IDs of the Ingester EC2 instances", + "value": "${aws_instance.ingester[*].id}" + }, + "ingester_public_ips": { + "description": "Public IP addresses of the Ingester EC2 instances", + "value": "${aws_instance.ingester[*].public_ip}" + }, + "ingester_private_ips": { + "description": "Private IP addresses of the Ingester EC2 instances", + "value": "${aws_instance.ingester[*].private_ip}" + }, + "signoz_instance_ids": { + "description": "IDs of the SigNoz EC2 instances", + "value": "${aws_instance.signoz[*].id}" + }, + "signoz_public_ips": { + "description": "Public IP addresses of the SigNoz EC2 instances", + "value": "${aws_instance.signoz[*].public_ip}" + }, + "signoz_private_ips": { + "description": "Private IP addresses of the SigNoz EC2 instances", + "value": "${aws_instance.signoz[*].private_ip}" + } + } +} diff --git a/internal/infrastructure/terraform/templates/aws/ec2/variables.tf.json.gotmpl b/internal/infrastructure/terraform/templates/aws/ec2/variables.tf.json.gotmpl new file mode 100644 index 0000000..1d1061d --- /dev/null +++ b/internal/infrastructure/terraform/templates/aws/ec2/variables.tf.json.gotmpl @@ -0,0 +1,79 @@ +{ + "variable": { + "aws_region": { + "description": "AWS region to deploy resources", + "type": "string", + "default": "us-east-1" + }, + "vpc_cidr": { + "description": "CIDR block for the VPC", + "type": "string", + "default": "10.0.0.0/16" + }, + "az_count": { + "description": "Number of availability zones to use", + "type": "number", + "default": 2 + }, + "name": { + "description": "The name of the deployment", + "type": "string", + "default": "{{ .Metadata.Name }}" + }, + "ssh_public_key": { + "description": "SSH public key for EC2 instance access. Leave empty to disable SSH key pair creation.", + "type": "string", + "default": "" + }, + "telemetrykeeper_instance_type": { + "description": "EC2 instance type for TelemetryKeeper", + "type": "string", + "default": "t3.small" + }, + "telemetrykeeper_volume_size": { + "description": "Root volume size (GB) for TelemetryKeeper instances", + "type": "number", + "default": 20 + }, + "telemetrystore_instance_type": { + "description": "EC2 instance type for TelemetryStore", + "type": "string", + "default": "r6i.xlarge" + }, + "telemetrystore_volume_size": { + "description": "Root volume size (GB) for TelemetryStore instances", + "type": "number", + "default": 100 + }, + "metastore_instance_type": { + "description": "EC2 instance type for MetaStore", + "type": "string", + "default": "t3.small" + }, + "metastore_volume_size": { + "description": "Root volume size (GB) for MetaStore instances", + "type": "number", + "default": 20 + }, + "ingester_instance_type": { + "description": "EC2 instance type for Ingester", + "type": "string", + "default": "t3.medium" + }, + "ingester_volume_size": { + "description": "Root volume size (GB) for Ingester instances", + "type": "number", + "default": 50 + }, + "signoz_instance_type": { + "description": "EC2 instance type for SigNoz", + "type": "string", + "default": "t3.medium" + }, + "signoz_volume_size": { + "description": "Root volume size (GB) for SigNoz instances", + "type": "number", + "default": 50 + } + } +} diff --git a/internal/infrastructure/terraform/templates/aws/eks/main.tf.json.gotmpl b/internal/infrastructure/terraform/templates/aws/eks/main.tf.json.gotmpl new file mode 100644 index 0000000..5376c04 --- /dev/null +++ b/internal/infrastructure/terraform/templates/aws/eks/main.tf.json.gotmpl @@ -0,0 +1,327 @@ +{ + "locals": { + "name": "{{ .Metadata.Name }}", + "common_tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "data": { + "aws_availability_zones": { + "available": { + "state": "available" + } + } + }, + "resource": { + "aws_vpc": { + "main": { + "cidr_block": "${var.vpc_cidr}", + "enable_dns_hostnames": true, + "enable_dns_support": true, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-vpc", + "kubernetes.io/cluster/${local.name}": "shared" + } + } + }, + "aws_subnet": { + "private": { + "count": "${var.az_count}", + "vpc_id": "${aws_vpc.main.id}", + "cidr_block": "${cidrsubnet(var.vpc_cidr, 8, count.index)}", + "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-private-${count.index}", + "kubernetes.io/cluster/${local.name}": "shared", + "kubernetes.io/role/internal-elb": "1" + } + }, + "public": { + "count": "${var.az_count}", + "vpc_id": "${aws_vpc.main.id}", + "cidr_block": "${cidrsubnet(var.vpc_cidr, 8, count.index + var.az_count)}", + "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", + "map_public_ip_on_launch": true, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-public-${count.index}", + "kubernetes.io/cluster/${local.name}": "shared", + "kubernetes.io/role/elb": "1" + } + } + }, + "aws_internet_gateway": { + "main": { + "vpc_id": "${aws_vpc.main.id}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-igw" + } + } + }, + "aws_eip": { + "nat": { + "count": "${var.az_count}", + "domain": "vpc", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-nat-eip-${count.index}" + } + } + }, + "aws_nat_gateway": { + "main": { + "count": "${var.az_count}", + "allocation_id": "${aws_eip.nat[count.index].id}", + "subnet_id": "${aws_subnet.public[count.index].id}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-nat-${count.index}" + }, + "depends_on": ["${aws_internet_gateway.main}"] + } + }, + "aws_route_table": { + "public": { + "vpc_id": "${aws_vpc.main.id}", + "route": [ + { + "cidr_block": "0.0.0.0/0", + "gateway_id": "${aws_internet_gateway.main.id}" + } + ], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-public-rt" + } + }, + "private": { + "count": "${var.az_count}", + "vpc_id": "${aws_vpc.main.id}", + "route": [ + { + "cidr_block": "0.0.0.0/0", + "nat_gateway_id": "${aws_nat_gateway.main[count.index].id}" + } + ], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-private-rt-${count.index}" + } + } + }, + "aws_route_table_association": { + "public": { + "count": "${var.az_count}", + "subnet_id": "${aws_subnet.public[count.index].id}", + "route_table_id": "${aws_route_table.public.id}" + }, + "private": { + "count": "${var.az_count}", + "subnet_id": "${aws_subnet.private[count.index].id}", + "route_table_id": "${aws_route_table.private[count.index].id}" + } + }, + "aws_iam_role": { + "eks_cluster": { + "name": "${local.name}-eks-cluster-role", + "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"eks.amazonaws.com\"}}]})}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "eks_node_group": { + "name": "${local.name}-eks-node-group-role", + "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"ec2.amazonaws.com\"}}]})}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + } + }, + "aws_iam_role_policy_attachment": { + "eks_cluster_policy": { + "policy_arn": "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", + "role": "${aws_iam_role.eks_cluster.name}" + }, + "eks_worker_node_policy": { + "policy_arn": "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + "role": "${aws_iam_role.eks_node_group.name}" + }, + "eks_cni_policy": { + "policy_arn": "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + "role": "${aws_iam_role.eks_node_group.name}" + }, + "eks_container_registry": { + "policy_arn": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + "role": "${aws_iam_role.eks_node_group.name}" + } + }, + "aws_eks_cluster": { + "main": { + "name": "${local.name}", + "role_arn": "${aws_iam_role.eks_cluster.arn}", + "version": "${var.kubernetes_version}", + "vpc_config": [{ + "subnet_ids": "${concat(aws_subnet.private[*].id, aws_subnet.public[*].id)}", + "endpoint_private_access": true, + "endpoint_public_access": true + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + }, + "depends_on": ["${aws_iam_role_policy_attachment.eks_cluster_policy}"] + } + }, + "aws_eks_node_group": { + "telemetrykeeper": { + "count": {{ if .Spec.TelemetryKeeper.Spec.Enabled }}1{{ else }}0{{ end }}, + "cluster_name": "${aws_eks_cluster.main.name}", + "node_group_name": "${local.name}-telemetrykeeper", + "node_role_arn": "${aws_iam_role.eks_node_group.arn}", + "subnet_ids": "${aws_subnet.private[*].id}", + "instance_types": ["${var.telemetrykeeper_instance_type}"], + "scaling_config": [{ + "desired_size": {{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }}, + "min_size": 1, + "max_size": {{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }} + }], + "disk_size": "${var.telemetrykeeper_volume_size}", + "labels": { + "role": "telemetrykeeper" + }, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-telemetrykeeper" + }, + "depends_on": [ + "${aws_iam_role_policy_attachment.eks_worker_node_policy}", + "${aws_iam_role_policy_attachment.eks_cni_policy}", + "${aws_iam_role_policy_attachment.eks_container_registry}" + ] + }, + "telemetrystore": { + "count": {{ if .Spec.TelemetryStore.Spec.Enabled }}1{{ else }}0{{ end }}, + "cluster_name": "${aws_eks_cluster.main.name}", + "node_group_name": "${local.name}-telemetrystore", + "node_role_arn": "${aws_iam_role.eks_node_group.arn}", + "subnet_ids": "${aws_subnet.private[*].id}", + "instance_types": ["${var.telemetrystore_instance_type}"], + "scaling_config": [{ + "desired_size": {{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }}, + "min_size": 1, + "max_size": {{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }} + }], + "disk_size": "${var.telemetrystore_volume_size}", + "labels": { + "role": "telemetrystore" + }, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-telemetrystore" + }, + "depends_on": [ + "${aws_iam_role_policy_attachment.eks_worker_node_policy}", + "${aws_iam_role_policy_attachment.eks_cni_policy}", + "${aws_iam_role_policy_attachment.eks_container_registry}" + ] + }, + "metastore": { + "count": {{ if .Spec.MetaStore.Spec.Enabled }}1{{ else }}0{{ end }}, + "cluster_name": "${aws_eks_cluster.main.name}", + "node_group_name": "${local.name}-metastore", + "node_role_arn": "${aws_iam_role.eks_node_group.arn}", + "subnet_ids": "${aws_subnet.private[*].id}", + "instance_types": ["${var.metastore_instance_type}"], + "scaling_config": [{ + "desired_size": {{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }}, + "min_size": 1, + "max_size": {{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }} + }], + "disk_size": "${var.metastore_volume_size}", + "labels": { + "role": "metastore" + }, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-metastore" + }, + "depends_on": [ + "${aws_iam_role_policy_attachment.eks_worker_node_policy}", + "${aws_iam_role_policy_attachment.eks_cni_policy}", + "${aws_iam_role_policy_attachment.eks_container_registry}" + ] + }, + "ingester": { + "count": {{ if .Spec.Ingester.Spec.Enabled }}1{{ else }}0{{ end }}, + "cluster_name": "${aws_eks_cluster.main.name}", + "node_group_name": "${local.name}-ingester", + "node_role_arn": "${aws_iam_role.eks_node_group.arn}", + "subnet_ids": "${aws_subnet.public[*].id}", + "instance_types": ["${var.ingester_instance_type}"], + "scaling_config": [{ + "desired_size": {{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}, + "min_size": 1, + "max_size": {{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }} + }], + "disk_size": "${var.ingester_volume_size}", + "labels": { + "role": "ingester" + }, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-ingester" + }, + "depends_on": [ + "${aws_iam_role_policy_attachment.eks_worker_node_policy}", + "${aws_iam_role_policy_attachment.eks_cni_policy}", + "${aws_iam_role_policy_attachment.eks_container_registry}" + ] + }, + "signoz": { + "count": {{ if .Spec.Signoz.Spec.Enabled }}1{{ else }}0{{ end }}, + "cluster_name": "${aws_eks_cluster.main.name}", + "node_group_name": "${local.name}-signoz", + "node_role_arn": "${aws_iam_role.eks_node_group.arn}", + "subnet_ids": "${aws_subnet.public[*].id}", + "instance_types": ["${var.signoz_instance_type}"], + "scaling_config": [{ + "desired_size": {{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}, + "min_size": 1, + "max_size": {{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }} + }], + "disk_size": "${var.signoz_volume_size}", + "labels": { + "role": "signoz" + }, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-signoz" + }, + "depends_on": [ + "${aws_iam_role_policy_attachment.eks_worker_node_policy}", + "${aws_iam_role_policy_attachment.eks_cni_policy}", + "${aws_iam_role_policy_attachment.eks_container_registry}" + ] + } + } + } +} diff --git a/internal/infrastructure/terraform/templates/aws/eks/outputs.tf.json.gotmpl b/internal/infrastructure/terraform/templates/aws/eks/outputs.tf.json.gotmpl new file mode 100644 index 0000000..e071577 --- /dev/null +++ b/internal/infrastructure/terraform/templates/aws/eks/outputs.tf.json.gotmpl @@ -0,0 +1,73 @@ +{ + "output": { + "cluster_name": { + "description": "Name of the EKS cluster", + "value": "${aws_eks_cluster.main.name}" + }, + "cluster_endpoint": { + "description": "Endpoint for the EKS cluster API server", + "value": "${aws_eks_cluster.main.endpoint}" + }, + "cluster_ca_certificate": { + "description": "Base64-encoded certificate authority data for the EKS cluster", + "value": "${aws_eks_cluster.main.certificate_authority[0].data}", + "sensitive": true + }, + "cluster_version": { + "description": "Kubernetes version of the EKS cluster", + "value": "${aws_eks_cluster.main.version}" + }, + "vpc_id": { + "description": "ID of the VPC", + "value": "${aws_vpc.main.id}" + }, + "private_subnet_ids": { + "description": "IDs of the private subnets", + "value": "${aws_subnet.private[*].id}" + }, + "public_subnet_ids": { + "description": "IDs of the public subnets", + "value": "${aws_subnet.public[*].id}" + }, + "telemetrykeeper_node_group_arn": { + "description": "ARN of the TelemetryKeeper node group", + "value": "${aws_eks_node_group.telemetrykeeper[*].arn}" + }, + "telemetrykeeper_node_group_status": { + "description": "Status of the TelemetryKeeper node group", + "value": "${aws_eks_node_group.telemetrykeeper[*].status}" + }, + "telemetrystore_node_group_arn": { + "description": "ARN of the TelemetryStore node group", + "value": "${aws_eks_node_group.telemetrystore[*].arn}" + }, + "telemetrystore_node_group_status": { + "description": "Status of the TelemetryStore node group", + "value": "${aws_eks_node_group.telemetrystore[*].status}" + }, + "metastore_node_group_arn": { + "description": "ARN of the MetaStore node group", + "value": "${aws_eks_node_group.metastore[*].arn}" + }, + "metastore_node_group_status": { + "description": "Status of the MetaStore node group", + "value": "${aws_eks_node_group.metastore[*].status}" + }, + "ingester_node_group_arn": { + "description": "ARN of the Ingester node group", + "value": "${aws_eks_node_group.ingester[*].arn}" + }, + "ingester_node_group_status": { + "description": "Status of the Ingester node group", + "value": "${aws_eks_node_group.ingester[*].status}" + }, + "signoz_node_group_arn": { + "description": "ARN of the SigNoz node group", + "value": "${aws_eks_node_group.signoz[*].arn}" + }, + "signoz_node_group_status": { + "description": "Status of the SigNoz node group", + "value": "${aws_eks_node_group.signoz[*].status}" + } + } +} diff --git a/internal/infrastructure/terraform/templates/aws/eks/variables.tf.json.gotmpl b/internal/infrastructure/terraform/templates/aws/eks/variables.tf.json.gotmpl new file mode 100644 index 0000000..47cfe41 --- /dev/null +++ b/internal/infrastructure/terraform/templates/aws/eks/variables.tf.json.gotmpl @@ -0,0 +1,79 @@ +{ + "variable": { + "aws_region": { + "description": "AWS region to deploy resources", + "type": "string", + "default": "us-east-1" + }, + "vpc_cidr": { + "description": "CIDR block for the VPC", + "type": "string", + "default": "10.0.0.0/16" + }, + "az_count": { + "description": "Number of availability zones to use", + "type": "number", + "default": 2 + }, + "name": { + "description": "The name of the deployment", + "type": "string", + "default": "{{ .Metadata.Name }}" + }, + "kubernetes_version": { + "description": "Kubernetes version for the EKS cluster", + "type": "string", + "default": "1.30" + }, + "telemetrykeeper_instance_type": { + "description": "EC2 instance type for TelemetryKeeper node group", + "type": "string", + "default": "t3.small" + }, + "telemetrykeeper_volume_size": { + "description": "Root volume size (GB) for TelemetryKeeper nodes", + "type": "number", + "default": 20 + }, + "telemetrystore_instance_type": { + "description": "EC2 instance type for TelemetryStore node group", + "type": "string", + "default": "r6i.xlarge" + }, + "telemetrystore_volume_size": { + "description": "Root volume size (GB) for TelemetryStore nodes", + "type": "number", + "default": 100 + }, + "metastore_instance_type": { + "description": "EC2 instance type for MetaStore node group", + "type": "string", + "default": "t3.small" + }, + "metastore_volume_size": { + "description": "Root volume size (GB) for MetaStore nodes", + "type": "number", + "default": 20 + }, + "ingester_instance_type": { + "description": "EC2 instance type for Ingester node group", + "type": "string", + "default": "t3.medium" + }, + "ingester_volume_size": { + "description": "Root volume size (GB) for Ingester nodes", + "type": "number", + "default": 50 + }, + "signoz_instance_type": { + "description": "EC2 instance type for SigNoz node group", + "type": "string", + "default": "t3.medium" + }, + "signoz_volume_size": { + "description": "Root volume size (GB) for SigNoz nodes", + "type": "number", + "default": 50 + } + } +} diff --git a/internal/infrastructure/terraform/templates/azure/aks/main.tf.json.gotmpl b/internal/infrastructure/terraform/templates/azure/aks/main.tf.json.gotmpl new file mode 100644 index 0000000..f0410ed --- /dev/null +++ b/internal/infrastructure/terraform/templates/azure/aks/main.tf.json.gotmpl @@ -0,0 +1,149 @@ +{ + "locals": { + "name": "{{ .Metadata.Name }}", + "common_tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "resource": { + "azurerm_resource_group": { + "main": { + "name": "${var.resource_group_name}", + "location": "${var.location}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + } + }, + "azurerm_virtual_network": { + "main": { + "name": "${local.name}-vnet", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "address_space": ["${var.vnet_cidr}"], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + } + }, + "azurerm_subnet": { + "aks": { + "name": "${local.name}-aks", + "resource_group_name": "${azurerm_resource_group.main.name}", + "virtual_network_name": "${azurerm_virtual_network.main.name}", + "address_prefixes": ["${var.aks_subnet_cidr}"] + } + }, + "azurerm_kubernetes_cluster": { + "main": { + "name": "${local.name}", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "dns_prefix": "${local.name}", + "kubernetes_version": "${var.kubernetes_version}", + "default_node_pool": [{ + "name": "system", + "node_count": 1, + "vm_size": "Standard_D2s_v3", + "vnet_subnet_id": "${azurerm_subnet.aks.id}", + "upgrade_settings": [{"max_surge": "10%"}] + }], + "identity": [{"type": "SystemAssigned"}], + "network_profile": [{ + "network_plugin": "azure", + "load_balancer_sku": "standard", + "outbound_type": "loadBalancer" + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + } + }, + "azurerm_kubernetes_cluster_node_pool": { + "telemetrykeeper": { + "count": {{ if .Spec.TelemetryKeeper.Spec.Enabled }}1{{ else }}0{{ end }}, + "name": "tkeepr", + "kubernetes_cluster_id": "${azurerm_kubernetes_cluster.main.id}", + "vm_size": "${var.telemetrykeeper_vm_size}", + "node_count": {{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }}, + "vnet_subnet_id": "${azurerm_subnet.aks.id}", + "os_disk_size_gb": "${var.telemetrykeeper_disk_size}", + "node_labels": { + "role": "telemetrykeeper" + }, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "telemetrystore": { + "count": {{ if .Spec.TelemetryStore.Spec.Enabled }}1{{ else }}0{{ end }}, + "name": "tstore", + "kubernetes_cluster_id": "${azurerm_kubernetes_cluster.main.id}", + "vm_size": "${var.telemetrystore_vm_size}", + "node_count": {{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }}, + "vnet_subnet_id": "${azurerm_subnet.aks.id}", + "os_disk_size_gb": "${var.telemetrystore_disk_size}", + "node_labels": { + "role": "telemetrystore" + }, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "metastore": { + "count": {{ if .Spec.MetaStore.Spec.Enabled }}1{{ else }}0{{ end }}, + "name": "metastore", + "kubernetes_cluster_id": "${azurerm_kubernetes_cluster.main.id}", + "vm_size": "${var.metastore_vm_size}", + "node_count": {{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }}, + "vnet_subnet_id": "${azurerm_subnet.aks.id}", + "os_disk_size_gb": "${var.metastore_disk_size}", + "node_labels": { + "role": "metastore" + }, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "ingester": { + "count": {{ if .Spec.Ingester.Spec.Enabled }}1{{ else }}0{{ end }}, + "name": "ingester", + "kubernetes_cluster_id": "${azurerm_kubernetes_cluster.main.id}", + "vm_size": "${var.ingester_vm_size}", + "node_count": {{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}, + "vnet_subnet_id": "${azurerm_subnet.aks.id}", + "os_disk_size_gb": "${var.ingester_disk_size}", + "node_labels": { + "role": "ingester" + }, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "signoz": { + "count": {{ if .Spec.Signoz.Spec.Enabled }}1{{ else }}0{{ end }}, + "name": "signoz", + "kubernetes_cluster_id": "${azurerm_kubernetes_cluster.main.id}", + "vm_size": "${var.signoz_vm_size}", + "node_count": {{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}, + "vnet_subnet_id": "${azurerm_subnet.aks.id}", + "os_disk_size_gb": "${var.signoz_disk_size}", + "node_labels": { + "role": "signoz" + }, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + } + } + } +} diff --git a/internal/infrastructure/terraform/templates/azure/aks/outputs.tf.json.gotmpl b/internal/infrastructure/terraform/templates/azure/aks/outputs.tf.json.gotmpl new file mode 100644 index 0000000..612cb03 --- /dev/null +++ b/internal/infrastructure/terraform/templates/azure/aks/outputs.tf.json.gotmpl @@ -0,0 +1,53 @@ +{ + "output": { + "cluster_name": { + "description": "Name of the AKS cluster", + "value": "${azurerm_kubernetes_cluster.main.name}" + }, + "cluster_fqdn": { + "description": "FQDN of the AKS cluster", + "value": "${azurerm_kubernetes_cluster.main.fqdn}" + }, + "kube_config": { + "description": "Raw kubeconfig for the AKS cluster", + "value": "${azurerm_kubernetes_cluster.main.kube_config_raw}", + "sensitive": true + }, + "cluster_identity_principal_id": { + "description": "Principal ID of the AKS cluster managed identity", + "value": "${azurerm_kubernetes_cluster.main.identity[0].principal_id}" + }, + "resource_group_name": { + "description": "Name of the Azure resource group", + "value": "${azurerm_resource_group.main.name}" + }, + "vnet_id": { + "description": "ID of the virtual network", + "value": "${azurerm_virtual_network.main.id}" + }, + "aks_subnet_id": { + "description": "ID of the AKS subnet", + "value": "${azurerm_subnet.aks.id}" + }, + "telemetrykeeper_node_pool_id": { + "description": "ID of the TelemetryKeeper node pool", + "value": "${azurerm_kubernetes_cluster_node_pool.telemetrykeeper[*].id}" + }, + "telemetrystore_node_pool_id": { + "description": "ID of the TelemetryStore node pool", + "value": "${azurerm_kubernetes_cluster_node_pool.telemetrystore[*].id}" + }, + "metastore_node_pool_id": { + "description": "ID of the MetaStore node pool", + "value": "${azurerm_kubernetes_cluster_node_pool.metastore[*].id}" + }, + "ingester_node_pool_id": { + "description": "ID of the Ingester node pool", + "value": "${azurerm_kubernetes_cluster_node_pool.ingester[*].id}" + }, + "signoz_node_pool_id": { + "description": "ID of the SigNoz node pool", + "value": "${azurerm_kubernetes_cluster_node_pool.signoz[*].id}" + } + } +} diff --git a/internal/infrastructure/terraform/templates/azure/aks/variables.tf.json.gotmpl b/internal/infrastructure/terraform/templates/azure/aks/variables.tf.json.gotmpl new file mode 100644 index 0000000..af39836 --- /dev/null +++ b/internal/infrastructure/terraform/templates/azure/aks/variables.tf.json.gotmpl @@ -0,0 +1,83 @@ +{ + "variable": { + "resource_group_name": { + "description": "Azure resource group name", + "type": "string" + }, + "location": { + "description": "Azure region to deploy resources", + "type": "string", + "default": "eastus" + }, + "name": { + "description": "The name of the deployment", + "type": "string", + "default": "{{ .Metadata.Name }}" + }, + "vnet_cidr": { + "description": "CIDR block for the virtual network", + "type": "string", + "default": "10.0.0.0/8" + }, + "aks_subnet_cidr": { + "description": "CIDR block for the AKS subnet", + "type": "string", + "default": "10.240.0.0/16" + }, + "kubernetes_version": { + "description": "Kubernetes version for the AKS cluster (leave empty for latest)", + "type": "string", + "default": "" + }, + "telemetrykeeper_vm_size": { + "description": "Azure VM size for TelemetryKeeper node pool", + "type": "string", + "default": "Standard_B2s" + }, + "telemetrykeeper_disk_size": { + "description": "OS disk size (GB) for TelemetryKeeper nodes", + "type": "number", + "default": 20 + }, + "telemetrystore_vm_size": { + "description": "Azure VM size for TelemetryStore node pool", + "type": "string", + "default": "Standard_E4s_v3" + }, + "telemetrystore_disk_size": { + "description": "OS disk size (GB) for TelemetryStore nodes", + "type": "number", + "default": 100 + }, + "metastore_vm_size": { + "description": "Azure VM size for MetaStore node pool", + "type": "string", + "default": "Standard_B2s" + }, + "metastore_disk_size": { + "description": "OS disk size (GB) for MetaStore nodes", + "type": "number", + "default": 20 + }, + "ingester_vm_size": { + "description": "Azure VM size for Ingester node pool", + "type": "string", + "default": "Standard_B4ms" + }, + "ingester_disk_size": { + "description": "OS disk size (GB) for Ingester nodes", + "type": "number", + "default": 50 + }, + "signoz_vm_size": { + "description": "Azure VM size for SigNoz node pool", + "type": "string", + "default": "Standard_B4ms" + }, + "signoz_disk_size": { + "description": "OS disk size (GB) for SigNoz nodes", + "type": "number", + "default": 50 + } + } +} diff --git a/internal/infrastructure/terraform/templates/azure/vm/main.tf.json.gotmpl b/internal/infrastructure/terraform/templates/azure/vm/main.tf.json.gotmpl new file mode 100644 index 0000000..c23f8c6 --- /dev/null +++ b/internal/infrastructure/terraform/templates/azure/vm/main.tf.json.gotmpl @@ -0,0 +1,537 @@ +{ + "locals": { + "name": "{{ .Metadata.Name }}", + "common_tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "resource": { + "azurerm_resource_group": { + "main": { + "name": "${var.resource_group_name}", + "location": "${var.location}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + } + }, + "azurerm_virtual_network": { + "main": { + "name": "${local.name}-vnet", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "address_space": ["${var.vnet_cidr}"], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + } + }, + "azurerm_subnet": { + "private": { + "name": "${local.name}-private", + "resource_group_name": "${azurerm_resource_group.main.name}", + "virtual_network_name": "${azurerm_virtual_network.main.name}", + "address_prefixes": ["${var.private_subnet_cidr}"] + }, + "public": { + "name": "${local.name}-public", + "resource_group_name": "${azurerm_resource_group.main.name}", + "virtual_network_name": "${azurerm_virtual_network.main.name}", + "address_prefixes": ["${var.public_subnet_cidr}"] + } + }, + "azurerm_network_security_group": { + "telemetrykeeper": { + "name": "${local.name}-telemetrykeeper-nsg", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "security_rule": [ + { + "name": "allow-keeper-client", + "priority": 100, + "direction": "Inbound", + "access": "Allow", + "protocol": "Tcp", + "source_port_range": "*", + "destination_port_range": "9181", + "source_address_prefix": "${var.vnet_cidr}", + "destination_address_prefix": "*" + }, + { + "name": "allow-keeper-raft", + "priority": 110, + "direction": "Inbound", + "access": "Allow", + "protocol": "Tcp", + "source_port_range": "*", + "destination_port_range": "9234", + "source_address_prefix": "${var.vnet_cidr}", + "destination_address_prefix": "*" + }, + { + "name": "allow-ssh", + "priority": 120, + "direction": "Inbound", + "access": "Allow", + "protocol": "Tcp", + "source_port_range": "*", + "destination_port_range": "22", + "source_address_prefix": "${var.vnet_cidr}", + "destination_address_prefix": "*" + } + ], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "telemetrystore": { + "name": "${local.name}-telemetrystore-nsg", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "security_rule": [ + { + "name": "allow-clickhouse-native", + "priority": 100, + "direction": "Inbound", + "access": "Allow", + "protocol": "Tcp", + "source_port_range": "*", + "destination_port_range": "9000", + "source_address_prefix": "${var.vnet_cidr}", + "destination_address_prefix": "*" + }, + { + "name": "allow-clickhouse-http", + "priority": 110, + "direction": "Inbound", + "access": "Allow", + "protocol": "Tcp", + "source_port_range": "*", + "destination_port_range": "8123", + "source_address_prefix": "${var.vnet_cidr}", + "destination_address_prefix": "*" + }, + { + "name": "allow-ssh", + "priority": 120, + "direction": "Inbound", + "access": "Allow", + "protocol": "Tcp", + "source_port_range": "*", + "destination_port_range": "22", + "source_address_prefix": "${var.vnet_cidr}", + "destination_address_prefix": "*" + } + ], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "metastore": { + "name": "${local.name}-metastore-nsg", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "security_rule": [ + { + "name": "allow-postgres", + "priority": 100, + "direction": "Inbound", + "access": "Allow", + "protocol": "Tcp", + "source_port_range": "*", + "destination_port_range": "5432", + "source_address_prefix": "${var.vnet_cidr}", + "destination_address_prefix": "*" + }, + { + "name": "allow-ssh", + "priority": 110, + "direction": "Inbound", + "access": "Allow", + "protocol": "Tcp", + "source_port_range": "*", + "destination_port_range": "22", + "source_address_prefix": "${var.vnet_cidr}", + "destination_address_prefix": "*" + } + ], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "ingester": { + "name": "${local.name}-ingester-nsg", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "security_rule": [ + { + "name": "allow-otlp-grpc", + "priority": 100, + "direction": "Inbound", + "access": "Allow", + "protocol": "Tcp", + "source_port_range": "*", + "destination_port_range": "4317", + "source_address_prefix": "*", + "destination_address_prefix": "*" + }, + { + "name": "allow-otlp-http", + "priority": 110, + "direction": "Inbound", + "access": "Allow", + "protocol": "Tcp", + "source_port_range": "*", + "destination_port_range": "4318", + "source_address_prefix": "*", + "destination_address_prefix": "*" + }, + { + "name": "allow-ssh", + "priority": 120, + "direction": "Inbound", + "access": "Allow", + "protocol": "Tcp", + "source_port_range": "*", + "destination_port_range": "22", + "source_address_prefix": "${var.vnet_cidr}", + "destination_address_prefix": "*" + } + ], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "signoz": { + "name": "${local.name}-signoz-nsg", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "security_rule": [ + { + "name": "allow-ui", + "priority": 100, + "direction": "Inbound", + "access": "Allow", + "protocol": "Tcp", + "source_port_range": "*", + "destination_port_range": "8080", + "source_address_prefix": "*", + "destination_address_prefix": "*" + }, + { + "name": "allow-api", + "priority": 110, + "direction": "Inbound", + "access": "Allow", + "protocol": "Tcp", + "source_port_range": "*", + "destination_port_range": "3301", + "source_address_prefix": "${var.vnet_cidr}", + "destination_address_prefix": "*" + }, + { + "name": "allow-ssh", + "priority": 120, + "direction": "Inbound", + "access": "Allow", + "protocol": "Tcp", + "source_port_range": "*", + "destination_port_range": "22", + "source_address_prefix": "${var.vnet_cidr}", + "destination_address_prefix": "*" + } + ], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + } + }, + "azurerm_network_interface": { + "telemetrykeeper": { + "count": {{ if .Spec.TelemetryKeeper.Spec.Enabled }}{{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "name": "${local.name}-telemetrykeeper-nic-${count.index}", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "ip_configuration": [{ + "name": "internal", + "subnet_id": "${azurerm_subnet.private.id}", + "private_ip_address_allocation": "Dynamic" + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "telemetrystore": { + "count": {{ if .Spec.TelemetryStore.Spec.Enabled }}{{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "name": "${local.name}-telemetrystore-nic-${count.index}", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "ip_configuration": [{ + "name": "internal", + "subnet_id": "${azurerm_subnet.private.id}", + "private_ip_address_allocation": "Dynamic" + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "metastore": { + "count": {{ if .Spec.MetaStore.Spec.Enabled }}{{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "name": "${local.name}-metastore-nic-${count.index}", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "ip_configuration": [{ + "name": "internal", + "subnet_id": "${azurerm_subnet.private.id}", + "private_ip_address_allocation": "Dynamic" + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "ingester": { + "count": {{ if .Spec.Ingester.Spec.Enabled }}{{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "name": "${local.name}-ingester-nic-${count.index}", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "ip_configuration": [{ + "name": "external", + "subnet_id": "${azurerm_subnet.public.id}", + "private_ip_address_allocation": "Dynamic", + "public_ip_address_id": "${azurerm_public_ip.ingester[count.index].id}" + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "signoz": { + "count": {{ if .Spec.Signoz.Spec.Enabled }}{{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "name": "${local.name}-signoz-nic-${count.index}", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "ip_configuration": [{ + "name": "external", + "subnet_id": "${azurerm_subnet.public.id}", + "private_ip_address_allocation": "Dynamic", + "public_ip_address_id": "${azurerm_public_ip.signoz[count.index].id}" + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + } + }, + "azurerm_network_interface_security_group_association": { + "telemetrykeeper": { + "count": {{ if .Spec.TelemetryKeeper.Spec.Enabled }}{{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "network_interface_id": "${azurerm_network_interface.telemetrykeeper[count.index].id}", + "network_security_group_id": "${azurerm_network_security_group.telemetrykeeper.id}" + }, + "telemetrystore": { + "count": {{ if .Spec.TelemetryStore.Spec.Enabled }}{{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "network_interface_id": "${azurerm_network_interface.telemetrystore[count.index].id}", + "network_security_group_id": "${azurerm_network_security_group.telemetrystore.id}" + }, + "metastore": { + "count": {{ if .Spec.MetaStore.Spec.Enabled }}{{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "network_interface_id": "${azurerm_network_interface.metastore[count.index].id}", + "network_security_group_id": "${azurerm_network_security_group.metastore.id}" + }, + "ingester": { + "count": {{ if .Spec.Ingester.Spec.Enabled }}{{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "network_interface_id": "${azurerm_network_interface.ingester[count.index].id}", + "network_security_group_id": "${azurerm_network_security_group.ingester.id}" + }, + "signoz": { + "count": {{ if .Spec.Signoz.Spec.Enabled }}{{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "network_interface_id": "${azurerm_network_interface.signoz[count.index].id}", + "network_security_group_id": "${azurerm_network_security_group.signoz.id}" + } + }, + "azurerm_public_ip": { + "ingester": { + "count": {{ if .Spec.Ingester.Spec.Enabled }}{{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "name": "${local.name}-ingester-pip-${count.index}", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "allocation_method": "Static", + "sku": "Standard", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "signoz": { + "count": {{ if .Spec.Signoz.Spec.Enabled }}{{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "name": "${local.name}-signoz-pip-${count.index}", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "allocation_method": "Static", + "sku": "Standard", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + } + }, + "azurerm_linux_virtual_machine": { + "telemetrykeeper": { + "count": {{ if .Spec.TelemetryKeeper.Spec.Enabled }}{{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "name": "${local.name}-telemetrykeeper-${count.index}", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "size": "${var.telemetrykeeper_vm_size}", + "admin_username": "ubuntu", + "network_interface_ids": ["${azurerm_network_interface.telemetrykeeper[count.index].id}"], + "admin_ssh_key": [{ + "username": "ubuntu", + "public_key": "${var.ssh_public_key}" + }], + "os_disk": [{ + "caching": "ReadWrite", + "storage_account_type": "Premium_LRS", + "disk_size_gb": "${var.telemetrykeeper_disk_size}" + }], + "source_image_reference": [{ + "publisher": "Canonical", + "offer": "0001-com-ubuntu-server-jammy", + "sku": "22_04-lts", + "version": "latest" + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Role": "telemetrykeeper" + } + }, + "telemetrystore": { + "count": {{ if .Spec.TelemetryStore.Spec.Enabled }}{{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "name": "${local.name}-telemetrystore-${count.index}", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "size": "${var.telemetrystore_vm_size}", + "admin_username": "ubuntu", + "network_interface_ids": ["${azurerm_network_interface.telemetrystore[count.index].id}"], + "admin_ssh_key": [{ + "username": "ubuntu", + "public_key": "${var.ssh_public_key}" + }], + "os_disk": [{ + "caching": "ReadWrite", + "storage_account_type": "Premium_LRS", + "disk_size_gb": "${var.telemetrystore_disk_size}" + }], + "source_image_reference": [{ + "publisher": "Canonical", + "offer": "0001-com-ubuntu-server-jammy", + "sku": "22_04-lts", + "version": "latest" + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Role": "telemetrystore" + } + }, + "metastore": { + "count": {{ if .Spec.MetaStore.Spec.Enabled }}{{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "name": "${local.name}-metastore-${count.index}", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "size": "${var.metastore_vm_size}", + "admin_username": "ubuntu", + "network_interface_ids": ["${azurerm_network_interface.metastore[count.index].id}"], + "admin_ssh_key": [{ + "username": "ubuntu", + "public_key": "${var.ssh_public_key}" + }], + "os_disk": [{ + "caching": "ReadWrite", + "storage_account_type": "Premium_LRS", + "disk_size_gb": "${var.metastore_disk_size}" + }], + "source_image_reference": [{ + "publisher": "Canonical", + "offer": "0001-com-ubuntu-server-jammy", + "sku": "22_04-lts", + "version": "latest" + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Role": "metastore" + } + }, + "ingester": { + "count": {{ if .Spec.Ingester.Spec.Enabled }}{{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "name": "${local.name}-ingester-${count.index}", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "size": "${var.ingester_vm_size}", + "admin_username": "ubuntu", + "network_interface_ids": ["${azurerm_network_interface.ingester[count.index].id}"], + "admin_ssh_key": [{ + "username": "ubuntu", + "public_key": "${var.ssh_public_key}" + }], + "os_disk": [{ + "caching": "ReadWrite", + "storage_account_type": "Premium_LRS", + "disk_size_gb": "${var.ingester_disk_size}" + }], + "source_image_reference": [{ + "publisher": "Canonical", + "offer": "0001-com-ubuntu-server-jammy", + "sku": "22_04-lts", + "version": "latest" + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Role": "ingester" + } + }, + "signoz": { + "count": {{ if .Spec.Signoz.Spec.Enabled }}{{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "name": "${local.name}-signoz-${count.index}", + "location": "${azurerm_resource_group.main.location}", + "resource_group_name": "${azurerm_resource_group.main.name}", + "size": "${var.signoz_vm_size}", + "admin_username": "ubuntu", + "network_interface_ids": ["${azurerm_network_interface.signoz[count.index].id}"], + "admin_ssh_key": [{ + "username": "ubuntu", + "public_key": "${var.ssh_public_key}" + }], + "os_disk": [{ + "caching": "ReadWrite", + "storage_account_type": "Premium_LRS", + "disk_size_gb": "${var.signoz_disk_size}" + }], + "source_image_reference": [{ + "publisher": "Canonical", + "offer": "0001-com-ubuntu-server-jammy", + "sku": "22_04-lts", + "version": "latest" + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Role": "signoz" + } + } + } + } +} diff --git a/internal/infrastructure/terraform/templates/azure/vm/outputs.tf.json.gotmpl b/internal/infrastructure/terraform/templates/azure/vm/outputs.tf.json.gotmpl new file mode 100644 index 0000000..152e95a --- /dev/null +++ b/internal/infrastructure/terraform/templates/azure/vm/outputs.tf.json.gotmpl @@ -0,0 +1,68 @@ +{ + "output": { + "resource_group_name": { + "description": "Name of the Azure resource group", + "value": "${azurerm_resource_group.main.name}" + }, + "vnet_id": { + "description": "ID of the virtual network", + "value": "${azurerm_virtual_network.main.id}" + }, + "private_subnet_id": { + "description": "ID of the private subnet", + "value": "${azurerm_subnet.private.id}" + }, + "public_subnet_id": { + "description": "ID of the public subnet", + "value": "${azurerm_subnet.public.id}" + }, + "telemetrykeeper_vm_ids": { + "description": "IDs of the TelemetryKeeper VMs", + "value": "${azurerm_linux_virtual_machine.telemetrykeeper[*].id}" + }, + "telemetrykeeper_private_ips": { + "description": "Private IP addresses of the TelemetryKeeper VMs", + "value": "${azurerm_network_interface.telemetrykeeper[*].private_ip_address}" + }, + "telemetrystore_vm_ids": { + "description": "IDs of the TelemetryStore VMs", + "value": "${azurerm_linux_virtual_machine.telemetrystore[*].id}" + }, + "telemetrystore_private_ips": { + "description": "Private IP addresses of the TelemetryStore VMs", + "value": "${azurerm_network_interface.telemetrystore[*].private_ip_address}" + }, + "metastore_vm_ids": { + "description": "IDs of the MetaStore VMs", + "value": "${azurerm_linux_virtual_machine.metastore[*].id}" + }, + "metastore_private_ips": { + "description": "Private IP addresses of the MetaStore VMs", + "value": "${azurerm_network_interface.metastore[*].private_ip_address}" + }, + "ingester_vm_ids": { + "description": "IDs of the Ingester VMs", + "value": "${azurerm_linux_virtual_machine.ingester[*].id}" + }, + "ingester_public_ips": { + "description": "Public IP addresses of the Ingester VMs", + "value": "${azurerm_public_ip.ingester[*].ip_address}" + }, + "ingester_private_ips": { + "description": "Private IP addresses of the Ingester VMs", + "value": "${azurerm_network_interface.ingester[*].private_ip_address}" + }, + "signoz_vm_ids": { + "description": "IDs of the SigNoz VMs", + "value": "${azurerm_linux_virtual_machine.signoz[*].id}" + }, + "signoz_public_ips": { + "description": "Public IP addresses of the SigNoz VMs", + "value": "${azurerm_public_ip.signoz[*].ip_address}" + }, + "signoz_private_ips": { + "description": "Private IP addresses of the SigNoz VMs", + "value": "${azurerm_network_interface.signoz[*].private_ip_address}" + } + } +} diff --git a/internal/infrastructure/terraform/templates/azure/vm/variables.tf.json.gotmpl b/internal/infrastructure/terraform/templates/azure/vm/variables.tf.json.gotmpl new file mode 100644 index 0000000..5326737 --- /dev/null +++ b/internal/infrastructure/terraform/templates/azure/vm/variables.tf.json.gotmpl @@ -0,0 +1,87 @@ +{ + "variable": { + "resource_group_name": { + "description": "Azure resource group name", + "type": "string" + }, + "location": { + "description": "Azure region to deploy resources", + "type": "string", + "default": "eastus" + }, + "name": { + "description": "The name of the deployment", + "type": "string", + "default": "{{ .Metadata.Name }}" + }, + "vnet_cidr": { + "description": "CIDR block for the virtual network", + "type": "string", + "default": "10.0.0.0/16" + }, + "private_subnet_cidr": { + "description": "CIDR block for the private subnet", + "type": "string", + "default": "10.0.1.0/24" + }, + "public_subnet_cidr": { + "description": "CIDR block for the public subnet", + "type": "string", + "default": "10.0.2.0/24" + }, + "ssh_public_key": { + "description": "SSH public key for VM access", + "type": "string" + }, + "telemetrykeeper_vm_size": { + "description": "Azure VM size for TelemetryKeeper", + "type": "string", + "default": "Standard_B2s" + }, + "telemetrykeeper_disk_size": { + "description": "OS disk size (GB) for TelemetryKeeper VMs", + "type": "number", + "default": 20 + }, + "telemetrystore_vm_size": { + "description": "Azure VM size for TelemetryStore", + "type": "string", + "default": "Standard_E4s_v3" + }, + "telemetrystore_disk_size": { + "description": "OS disk size (GB) for TelemetryStore VMs", + "type": "number", + "default": 100 + }, + "metastore_vm_size": { + "description": "Azure VM size for MetaStore", + "type": "string", + "default": "Standard_B2s" + }, + "metastore_disk_size": { + "description": "OS disk size (GB) for MetaStore VMs", + "type": "number", + "default": 20 + }, + "ingester_vm_size": { + "description": "Azure VM size for Ingester", + "type": "string", + "default": "Standard_B4ms" + }, + "ingester_disk_size": { + "description": "OS disk size (GB) for Ingester VMs", + "type": "number", + "default": 50 + }, + "signoz_vm_size": { + "description": "Azure VM size for SigNoz", + "type": "string", + "default": "Standard_B4ms" + }, + "signoz_disk_size": { + "description": "OS disk size (GB) for SigNoz VMs", + "type": "number", + "default": 50 + } + } +} diff --git a/internal/infrastructure/terraform/templates/gcp/gce/main.tf.json.gotmpl b/internal/infrastructure/terraform/templates/gcp/gce/main.tf.json.gotmpl new file mode 100644 index 0000000..b0f5272 --- /dev/null +++ b/internal/infrastructure/terraform/templates/gcp/gce/main.tf.json.gotmpl @@ -0,0 +1,261 @@ +{ + "locals": { + "name": "{{ .Metadata.Name }}", + "common_labels": { + "app-kubernetes-io-managed-by": "foundry", + "app-kubernetes-io-name": "{{ .Metadata.Name }}" + } + }, + "data": { + "google_compute_image": { + "ubuntu": { + "family": "ubuntu-2204-lts", + "project": "ubuntu-os-cloud" + } + } + }, + "resource": { + "google_compute_network": { + "main": { + "name": "${local.name}-vpc", + "auto_create_subnetworks": false, + "project": "${var.project_id}" + } + }, + "google_compute_subnetwork": { + "private": { + "name": "${local.name}-private", + "ip_cidr_range": "${var.private_subnet_cidr}", + "region": "${var.region}", + "network": "${google_compute_network.main.id}", + "project": "${var.project_id}", + "private_ip_google_access": true + }, + "public": { + "name": "${local.name}-public", + "ip_cidr_range": "${var.public_subnet_cidr}", + "region": "${var.region}", + "network": "${google_compute_network.main.id}", + "project": "${var.project_id}" + } + }, + "google_compute_router": { + "main": { + "name": "${local.name}-router", + "region": "${var.region}", + "network": "${google_compute_network.main.id}", + "project": "${var.project_id}" + } + }, + "google_compute_router_nat": { + "main": { + "name": "${local.name}-nat", + "router": "${google_compute_router.main.name}", + "region": "${var.region}", + "project": "${var.project_id}", + "nat_ip_allocate_option": "AUTO_ONLY", + "source_subnetwork_ip_ranges_to_nat": "LIST_OF_SUBNETWORKS", + "subnetwork": [{ + "name": "${google_compute_subnetwork.private.id}", + "source_ip_ranges_to_nat": ["ALL_IP_RANGES"] + }] + } + }, + "google_compute_firewall": { + "telemetrykeeper": { + "name": "${local.name}-telemetrykeeper", + "network": "${google_compute_network.main.id}", + "project": "${var.project_id}", + "allow": [ + {"protocol": "tcp", "ports": ["9181", "9234"]}, + {"protocol": "tcp", "ports": ["22"]} + ], + "source_ranges": ["${var.private_subnet_cidr}"], + "target_tags": ["telemetrykeeper"], + "description": "Allow TelemetryKeeper (ClickHouse Keeper) traffic" + }, + "telemetrystore": { + "name": "${local.name}-telemetrystore", + "network": "${google_compute_network.main.id}", + "project": "${var.project_id}", + "allow": [ + {"protocol": "tcp", "ports": ["9000", "8123"]}, + {"protocol": "tcp", "ports": ["22"]} + ], + "source_ranges": ["${var.private_subnet_cidr}"], + "target_tags": ["telemetrystore"], + "description": "Allow TelemetryStore (ClickHouse) traffic" + }, + "metastore": { + "name": "${local.name}-metastore", + "network": "${google_compute_network.main.id}", + "project": "${var.project_id}", + "allow": [ + {"protocol": "tcp", "ports": ["5432"]}, + {"protocol": "tcp", "ports": ["22"]} + ], + "source_ranges": ["${var.private_subnet_cidr}"], + "target_tags": ["metastore"], + "description": "Allow MetaStore (PostgreSQL) traffic" + }, + "ingester": { + "name": "${local.name}-ingester", + "network": "${google_compute_network.main.id}", + "project": "${var.project_id}", + "allow": [ + {"protocol": "tcp", "ports": ["4317", "4318"]}, + {"protocol": "tcp", "ports": ["22"]} + ], + "source_ranges": ["0.0.0.0/0"], + "target_tags": ["ingester"], + "description": "Allow Ingester (OpenTelemetry Collector) traffic" + }, + "signoz": { + "name": "${local.name}-signoz", + "network": "${google_compute_network.main.id}", + "project": "${var.project_id}", + "allow": [ + {"protocol": "tcp", "ports": ["8080", "3301"]}, + {"protocol": "tcp", "ports": ["22"]} + ], + "source_ranges": ["0.0.0.0/0"], + "target_tags": ["signoz"], + "description": "Allow SigNoz UI and API traffic" + } + }, + "google_compute_instance": { + "telemetrykeeper": { + "count": {{ if .Spec.TelemetryKeeper.Spec.Enabled }}{{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "name": "${local.name}-telemetrykeeper-${count.index}", + "machine_type": "${var.telemetrykeeper_machine_type}", + "zone": "${var.region}-${element([\"a\", \"b\", \"c\"], count.index)}", + "project": "${var.project_id}", + "tags": ["telemetrykeeper"], + "boot_disk": [{ + "initialize_params": [{ + "image": "${data.google_compute_image.ubuntu.self_link}", + "size": "${var.telemetrykeeper_disk_size}", + "type": "pd-ssd" + }] + }], + "network_interface": [{ + "subnetwork": "${google_compute_subnetwork.private.id}" + }], + "labels": { + "app-kubernetes-io-managed-by": "foundry", + "app-kubernetes-io-name": "{{ .Metadata.Name }}", + "role": "telemetrykeeper" + }, + "metadata": { + "enable-oslogin": "TRUE" + } + }, + "telemetrystore": { + "count": {{ if .Spec.TelemetryStore.Spec.Enabled }}{{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "name": "${local.name}-telemetrystore-${count.index}", + "machine_type": "${var.telemetrystore_machine_type}", + "zone": "${var.region}-${element([\"a\", \"b\", \"c\"], count.index)}", + "project": "${var.project_id}", + "tags": ["telemetrystore"], + "boot_disk": [{ + "initialize_params": [{ + "image": "${data.google_compute_image.ubuntu.self_link}", + "size": "${var.telemetrystore_disk_size}", + "type": "pd-ssd" + }] + }], + "network_interface": [{ + "subnetwork": "${google_compute_subnetwork.private.id}" + }], + "labels": { + "app-kubernetes-io-managed-by": "foundry", + "app-kubernetes-io-name": "{{ .Metadata.Name }}", + "role": "telemetrystore" + }, + "metadata": { + "enable-oslogin": "TRUE" + } + }, + "metastore": { + "count": {{ if .Spec.MetaStore.Spec.Enabled }}{{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "name": "${local.name}-metastore-${count.index}", + "machine_type": "${var.metastore_machine_type}", + "zone": "${var.region}-${element([\"a\", \"b\", \"c\"], count.index)}", + "project": "${var.project_id}", + "tags": ["metastore"], + "boot_disk": [{ + "initialize_params": [{ + "image": "${data.google_compute_image.ubuntu.self_link}", + "size": "${var.metastore_disk_size}", + "type": "pd-ssd" + }] + }], + "network_interface": [{ + "subnetwork": "${google_compute_subnetwork.private.id}" + }], + "labels": { + "app-kubernetes-io-managed-by": "foundry", + "app-kubernetes-io-name": "{{ .Metadata.Name }}", + "role": "metastore" + }, + "metadata": { + "enable-oslogin": "TRUE" + } + }, + "ingester": { + "count": {{ if .Spec.Ingester.Spec.Enabled }}{{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "name": "${local.name}-ingester-${count.index}", + "machine_type": "${var.ingester_machine_type}", + "zone": "${var.region}-${element([\"a\", \"b\", \"c\"], count.index)}", + "project": "${var.project_id}", + "tags": ["ingester"], + "boot_disk": [{ + "initialize_params": [{ + "image": "${data.google_compute_image.ubuntu.self_link}", + "size": "${var.ingester_disk_size}", + "type": "pd-ssd" + }] + }], + "network_interface": [{ + "subnetwork": "${google_compute_subnetwork.public.id}", + "access_config": [{}] + }], + "labels": { + "app-kubernetes-io-managed-by": "foundry", + "app-kubernetes-io-name": "{{ .Metadata.Name }}", + "role": "ingester" + }, + "metadata": { + "enable-oslogin": "TRUE" + } + }, + "signoz": { + "count": {{ if .Spec.Signoz.Spec.Enabled }}{{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, + "name": "${local.name}-signoz-${count.index}", + "machine_type": "${var.signoz_machine_type}", + "zone": "${var.region}-${element([\"a\", \"b\", \"c\"], count.index)}", + "project": "${var.project_id}", + "tags": ["signoz"], + "boot_disk": [{ + "initialize_params": [{ + "image": "${data.google_compute_image.ubuntu.self_link}", + "size": "${var.signoz_disk_size}", + "type": "pd-ssd" + }] + }], + "network_interface": [{ + "subnetwork": "${google_compute_subnetwork.public.id}", + "access_config": [{}] + }], + "labels": { + "app-kubernetes-io-managed-by": "foundry", + "app-kubernetes-io-name": "{{ .Metadata.Name }}", + "role": "signoz" + }, + "metadata": { + "enable-oslogin": "TRUE" + } + } + } + } +} diff --git a/internal/infrastructure/terraform/templates/gcp/gce/outputs.tf.json.gotmpl b/internal/infrastructure/terraform/templates/gcp/gce/outputs.tf.json.gotmpl new file mode 100644 index 0000000..abe8005 --- /dev/null +++ b/internal/infrastructure/terraform/templates/gcp/gce/outputs.tf.json.gotmpl @@ -0,0 +1,64 @@ +{ + "output": { + "network_id": { + "description": "ID of the VPC network", + "value": "${google_compute_network.main.id}" + }, + "private_subnetwork_id": { + "description": "ID of the private subnetwork", + "value": "${google_compute_subnetwork.private.id}" + }, + "public_subnetwork_id": { + "description": "ID of the public subnetwork", + "value": "${google_compute_subnetwork.public.id}" + }, + "telemetrykeeper_instance_ids": { + "description": "IDs of the TelemetryKeeper GCE instances", + "value": "${google_compute_instance.telemetrykeeper[*].id}" + }, + "telemetrykeeper_private_ips": { + "description": "Private IP addresses of the TelemetryKeeper GCE instances", + "value": "${[for i in google_compute_instance.telemetrykeeper : i.network_interface[0].network_ip]}" + }, + "telemetrystore_instance_ids": { + "description": "IDs of the TelemetryStore GCE instances", + "value": "${google_compute_instance.telemetrystore[*].id}" + }, + "telemetrystore_private_ips": { + "description": "Private IP addresses of the TelemetryStore GCE instances", + "value": "${[for i in google_compute_instance.telemetrystore : i.network_interface[0].network_ip]}" + }, + "metastore_instance_ids": { + "description": "IDs of the MetaStore GCE instances", + "value": "${google_compute_instance.metastore[*].id}" + }, + "metastore_private_ips": { + "description": "Private IP addresses of the MetaStore GCE instances", + "value": "${[for i in google_compute_instance.metastore : i.network_interface[0].network_ip]}" + }, + "ingester_instance_ids": { + "description": "IDs of the Ingester GCE instances", + "value": "${google_compute_instance.ingester[*].id}" + }, + "ingester_public_ips": { + "description": "Public IP addresses of the Ingester GCE instances", + "value": "${[for i in google_compute_instance.ingester : i.network_interface[0].access_config[0].nat_ip]}" + }, + "ingester_private_ips": { + "description": "Private IP addresses of the Ingester GCE instances", + "value": "${[for i in google_compute_instance.ingester : i.network_interface[0].network_ip]}" + }, + "signoz_instance_ids": { + "description": "IDs of the SigNoz GCE instances", + "value": "${google_compute_instance.signoz[*].id}" + }, + "signoz_public_ips": { + "description": "Public IP addresses of the SigNoz GCE instances", + "value": "${[for i in google_compute_instance.signoz : i.network_interface[0].access_config[0].nat_ip]}" + }, + "signoz_private_ips": { + "description": "Private IP addresses of the SigNoz GCE instances", + "value": "${[for i in google_compute_instance.signoz : i.network_interface[0].network_ip]}" + } + } +} diff --git a/internal/infrastructure/terraform/templates/gcp/gce/variables.tf.json.gotmpl b/internal/infrastructure/terraform/templates/gcp/gce/variables.tf.json.gotmpl new file mode 100644 index 0000000..c94e307 --- /dev/null +++ b/internal/infrastructure/terraform/templates/gcp/gce/variables.tf.json.gotmpl @@ -0,0 +1,78 @@ +{ + "variable": { + "project_id": { + "description": "GCP project ID", + "type": "string" + }, + "region": { + "description": "GCP region to deploy resources", + "type": "string", + "default": "us-central1" + }, + "name": { + "description": "The name of the deployment", + "type": "string", + "default": "{{ .Metadata.Name }}" + }, + "private_subnet_cidr": { + "description": "CIDR block for the private subnet", + "type": "string", + "default": "10.0.1.0/24" + }, + "public_subnet_cidr": { + "description": "CIDR block for the public subnet", + "type": "string", + "default": "10.0.2.0/24" + }, + "telemetrykeeper_machine_type": { + "description": "GCE machine type for TelemetryKeeper", + "type": "string", + "default": "n2-standard-2" + }, + "telemetrykeeper_disk_size": { + "description": "Boot disk size (GB) for TelemetryKeeper instances", + "type": "number", + "default": 20 + }, + "telemetrystore_machine_type": { + "description": "GCE machine type for TelemetryStore", + "type": "string", + "default": "n2-highmem-4" + }, + "telemetrystore_disk_size": { + "description": "Boot disk size (GB) for TelemetryStore instances", + "type": "number", + "default": 100 + }, + "metastore_machine_type": { + "description": "GCE machine type for MetaStore", + "type": "string", + "default": "n2-standard-2" + }, + "metastore_disk_size": { + "description": "Boot disk size (GB) for MetaStore instances", + "type": "number", + "default": 20 + }, + "ingester_machine_type": { + "description": "GCE machine type for Ingester", + "type": "string", + "default": "n2-standard-4" + }, + "ingester_disk_size": { + "description": "Boot disk size (GB) for Ingester instances", + "type": "number", + "default": 50 + }, + "signoz_machine_type": { + "description": "GCE machine type for SigNoz", + "type": "string", + "default": "n2-standard-4" + }, + "signoz_disk_size": { + "description": "Boot disk size (GB) for SigNoz instances", + "type": "number", + "default": 50 + } + } +} diff --git a/internal/infrastructure/terraform/templates/gcp/gke/main.tf.json.gotmpl b/internal/infrastructure/terraform/templates/gcp/gke/main.tf.json.gotmpl new file mode 100644 index 0000000..9a4a60d --- /dev/null +++ b/internal/infrastructure/terraform/templates/gcp/gke/main.tf.json.gotmpl @@ -0,0 +1,224 @@ +{ + "locals": { + "name": "{{ .Metadata.Name }}", + "common_labels": { + "app-kubernetes-io-managed-by": "foundry", + "app-kubernetes-io-name": "{{ .Metadata.Name }}" + } + }, + "resource": { + "google_compute_network": { + "main": { + "name": "${local.name}-vpc", + "auto_create_subnetworks": false, + "project": "${var.project_id}" + } + }, + "google_compute_subnetwork": { + "private": { + "name": "${local.name}-private", + "ip_cidr_range": "${var.private_subnet_cidr}", + "region": "${var.region}", + "network": "${google_compute_network.main.id}", + "project": "${var.project_id}", + "private_ip_google_access": true, + "secondary_ip_range": [ + { + "range_name": "${local.name}-pods", + "ip_cidr_range": "${var.pods_cidr}" + }, + { + "range_name": "${local.name}-services", + "ip_cidr_range": "${var.services_cidr}" + } + ] + }, + "public": { + "name": "${local.name}-public", + "ip_cidr_range": "${var.public_subnet_cidr}", + "region": "${var.region}", + "network": "${google_compute_network.main.id}", + "project": "${var.project_id}" + } + }, + "google_compute_router": { + "main": { + "name": "${local.name}-router", + "region": "${var.region}", + "network": "${google_compute_network.main.id}", + "project": "${var.project_id}" + } + }, + "google_compute_router_nat": { + "main": { + "name": "${local.name}-nat", + "router": "${google_compute_router.main.name}", + "region": "${var.region}", + "project": "${var.project_id}", + "nat_ip_allocate_option": "AUTO_ONLY", + "source_subnetwork_ip_ranges_to_nat": "LIST_OF_SUBNETWORKS", + "subnetwork": [{ + "name": "${google_compute_subnetwork.private.id}", + "source_ip_ranges_to_nat": ["ALL_IP_RANGES"] + }] + } + }, + "google_container_cluster": { + "main": { + "name": "${local.name}", + "location": "${var.region}", + "project": "${var.project_id}", + "network": "${google_compute_network.main.id}", + "subnetwork": "${google_compute_subnetwork.private.id}", + "remove_default_node_pool": true, + "initial_node_count": 1, + "ip_allocation_policy": [{ + "cluster_secondary_range_name": "${local.name}-pods", + "services_secondary_range_name": "${local.name}-services" + }], + "private_cluster_config": [{ + "enable_private_nodes": true, + "enable_private_endpoint": false, + "master_ipv4_cidr_block": "${var.master_cidr}" + }], + "master_auth": [{ + "client_certificate_config": [{ + "issue_client_certificate": false + }] + }], + "workload_identity_config": [{ + "workload_pool": "${var.project_id}.svc.id.goog" + }], + "release_channel": [{ + "channel": "REGULAR" + }], + "resource_labels": { + "app-kubernetes-io-managed-by": "foundry", + "app-kubernetes-io-name": "{{ .Metadata.Name }}" + } + } + }, + "google_container_node_pool": { + "telemetrykeeper": { + "count": {{ if .Spec.TelemetryKeeper.Spec.Enabled }}1{{ else }}0{{ end }}, + "name": "${local.name}-telemetrykeeper", + "cluster": "${google_container_cluster.main.name}", + "location": "${var.region}", + "project": "${var.project_id}", + "node_count": {{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }}, + "node_config": [{ + "machine_type": "${var.telemetrykeeper_machine_type}", + "disk_size_gb": "${var.telemetrykeeper_disk_size}", + "disk_type": "pd-ssd", + "labels": { + "app-kubernetes-io-managed-by": "foundry", + "app-kubernetes-io-name": "{{ .Metadata.Name }}", + "role": "telemetrykeeper" + }, + "workload_metadata_config": [{"mode": "GKE_METADATA"}], + "shielded_instance_config": [{"enable_secure_boot": true}] + }], + "management": [{ + "auto_repair": true, + "auto_upgrade": true + }] + }, + "telemetrystore": { + "count": {{ if .Spec.TelemetryStore.Spec.Enabled }}1{{ else }}0{{ end }}, + "name": "${local.name}-telemetrystore", + "cluster": "${google_container_cluster.main.name}", + "location": "${var.region}", + "project": "${var.project_id}", + "node_count": {{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }}, + "node_config": [{ + "machine_type": "${var.telemetrystore_machine_type}", + "disk_size_gb": "${var.telemetrystore_disk_size}", + "disk_type": "pd-ssd", + "labels": { + "app-kubernetes-io-managed-by": "foundry", + "app-kubernetes-io-name": "{{ .Metadata.Name }}", + "role": "telemetrystore" + }, + "workload_metadata_config": [{"mode": "GKE_METADATA"}], + "shielded_instance_config": [{"enable_secure_boot": true}] + }], + "management": [{ + "auto_repair": true, + "auto_upgrade": true + }] + }, + "metastore": { + "count": {{ if .Spec.MetaStore.Spec.Enabled }}1{{ else }}0{{ end }}, + "name": "${local.name}-metastore", + "cluster": "${google_container_cluster.main.name}", + "location": "${var.region}", + "project": "${var.project_id}", + "node_count": {{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }}, + "node_config": [{ + "machine_type": "${var.metastore_machine_type}", + "disk_size_gb": "${var.metastore_disk_size}", + "disk_type": "pd-ssd", + "labels": { + "app-kubernetes-io-managed-by": "foundry", + "app-kubernetes-io-name": "{{ .Metadata.Name }}", + "role": "metastore" + }, + "workload_metadata_config": [{"mode": "GKE_METADATA"}], + "shielded_instance_config": [{"enable_secure_boot": true}] + }], + "management": [{ + "auto_repair": true, + "auto_upgrade": true + }] + }, + "ingester": { + "count": {{ if .Spec.Ingester.Spec.Enabled }}1{{ else }}0{{ end }}, + "name": "${local.name}-ingester", + "cluster": "${google_container_cluster.main.name}", + "location": "${var.region}", + "project": "${var.project_id}", + "node_count": {{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}, + "node_config": [{ + "machine_type": "${var.ingester_machine_type}", + "disk_size_gb": "${var.ingester_disk_size}", + "disk_type": "pd-ssd", + "labels": { + "app-kubernetes-io-managed-by": "foundry", + "app-kubernetes-io-name": "{{ .Metadata.Name }}", + "role": "ingester" + }, + "workload_metadata_config": [{"mode": "GKE_METADATA"}], + "shielded_instance_config": [{"enable_secure_boot": true}] + }], + "management": [{ + "auto_repair": true, + "auto_upgrade": true + }] + }, + "signoz": { + "count": {{ if .Spec.Signoz.Spec.Enabled }}1{{ else }}0{{ end }}, + "name": "${local.name}-signoz", + "cluster": "${google_container_cluster.main.name}", + "location": "${var.region}", + "project": "${var.project_id}", + "node_count": {{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}, + "node_config": [{ + "machine_type": "${var.signoz_machine_type}", + "disk_size_gb": "${var.signoz_disk_size}", + "disk_type": "pd-ssd", + "labels": { + "app-kubernetes-io-managed-by": "foundry", + "app-kubernetes-io-name": "{{ .Metadata.Name }}", + "role": "signoz" + }, + "workload_metadata_config": [{"mode": "GKE_METADATA"}], + "shielded_instance_config": [{"enable_secure_boot": true}] + }], + "management": [{ + "auto_repair": true, + "auto_upgrade": true + }] + } + } + } +} diff --git a/internal/infrastructure/terraform/templates/gcp/gke/outputs.tf.json.gotmpl b/internal/infrastructure/terraform/templates/gcp/gke/outputs.tf.json.gotmpl new file mode 100644 index 0000000..e68728a --- /dev/null +++ b/internal/infrastructure/terraform/templates/gcp/gke/outputs.tf.json.gotmpl @@ -0,0 +1,46 @@ +{ + "output": { + "cluster_name": { + "description": "Name of the GKE cluster", + "value": "${google_container_cluster.main.name}" + }, + "cluster_endpoint": { + "description": "Endpoint for the GKE cluster master", + "value": "${google_container_cluster.main.endpoint}", + "sensitive": true + }, + "cluster_ca_certificate": { + "description": "Base64-encoded public certificate of the cluster's certificate authority", + "value": "${google_container_cluster.main.master_auth[0].cluster_ca_certificate}", + "sensitive": true + }, + "network_id": { + "description": "ID of the VPC network", + "value": "${google_compute_network.main.id}" + }, + "private_subnetwork_id": { + "description": "ID of the private subnetwork", + "value": "${google_compute_subnetwork.private.id}" + }, + "telemetrykeeper_node_pool_id": { + "description": "ID of the TelemetryKeeper node pool", + "value": "${google_container_node_pool.telemetrykeeper[*].id}" + }, + "telemetrystore_node_pool_id": { + "description": "ID of the TelemetryStore node pool", + "value": "${google_container_node_pool.telemetrystore[*].id}" + }, + "metastore_node_pool_id": { + "description": "ID of the MetaStore node pool", + "value": "${google_container_node_pool.metastore[*].id}" + }, + "ingester_node_pool_id": { + "description": "ID of the Ingester node pool", + "value": "${google_container_node_pool.ingester[*].id}" + }, + "signoz_node_pool_id": { + "description": "ID of the SigNoz node pool", + "value": "${google_container_node_pool.signoz[*].id}" + } + } +} diff --git a/internal/infrastructure/terraform/templates/gcp/gke/variables.tf.json.gotmpl b/internal/infrastructure/terraform/templates/gcp/gke/variables.tf.json.gotmpl new file mode 100644 index 0000000..6138d3c --- /dev/null +++ b/internal/infrastructure/terraform/templates/gcp/gke/variables.tf.json.gotmpl @@ -0,0 +1,98 @@ +{ + "variable": { + "project_id": { + "description": "GCP project ID", + "type": "string" + }, + "region": { + "description": "GCP region to deploy resources", + "type": "string", + "default": "us-central1" + }, + "name": { + "description": "The name of the deployment", + "type": "string", + "default": "{{ .Metadata.Name }}" + }, + "private_subnet_cidr": { + "description": "CIDR block for the private subnet", + "type": "string", + "default": "10.0.1.0/24" + }, + "public_subnet_cidr": { + "description": "CIDR block for the public subnet", + "type": "string", + "default": "10.0.2.0/24" + }, + "pods_cidr": { + "description": "Secondary CIDR block for GKE pods", + "type": "string", + "default": "10.1.0.0/16" + }, + "services_cidr": { + "description": "Secondary CIDR block for GKE services", + "type": "string", + "default": "10.2.0.0/20" + }, + "master_cidr": { + "description": "CIDR block for the GKE master nodes (must be /28)", + "type": "string", + "default": "172.16.0.0/28" + }, + "kubernetes_version": { + "description": "Minimum Kubernetes version for the GKE cluster (leave empty for latest)", + "type": "string", + "default": "" + }, + "telemetrykeeper_machine_type": { + "description": "GCE machine type for TelemetryKeeper node pool", + "type": "string", + "default": "n2-standard-2" + }, + "telemetrykeeper_disk_size": { + "description": "Boot disk size (GB) for TelemetryKeeper nodes", + "type": "number", + "default": 20 + }, + "telemetrystore_machine_type": { + "description": "GCE machine type for TelemetryStore node pool", + "type": "string", + "default": "n2-highmem-4" + }, + "telemetrystore_disk_size": { + "description": "Boot disk size (GB) for TelemetryStore nodes", + "type": "number", + "default": 100 + }, + "metastore_machine_type": { + "description": "GCE machine type for MetaStore node pool", + "type": "string", + "default": "n2-standard-2" + }, + "metastore_disk_size": { + "description": "Boot disk size (GB) for MetaStore nodes", + "type": "number", + "default": 20 + }, + "ingester_machine_type": { + "description": "GCE machine type for Ingester node pool", + "type": "string", + "default": "n2-standard-4" + }, + "ingester_disk_size": { + "description": "Boot disk size (GB) for Ingester nodes", + "type": "number", + "default": 50 + }, + "signoz_machine_type": { + "description": "GCE machine type for SigNoz node pool", + "type": "string", + "default": "n2-standard-4" + }, + "signoz_disk_size": { + "description": "Boot disk size (GB) for SigNoz nodes", + "type": "number", + "default": 50 + } + } +} diff --git a/internal/infrastructure/terraform/templates/providers.tf.json.gotmpl b/internal/infrastructure/terraform/templates/providers.tf.json.gotmpl new file mode 100644 index 0000000..cdcb200 --- /dev/null +++ b/internal/infrastructure/terraform/templates/providers.tf.json.gotmpl @@ -0,0 +1,32 @@ +{ + "terraform": { + "required_version": ">= 1.0.0", + "required_providers": { + {{- if eq .Provider.String "aws" }} + "aws": { + "source": "hashicorp/aws", + "version": "~> 5.0" + } + {{- else if eq .Provider.String "gcp" }} + "google": { + "source": "hashicorp/google", + "version": "~> 5.0" + } + {{- else if eq .Provider.String "azure" }} + "azurerm": { + "source": "hashicorp/azurerm", + "version": "~> 3.0" + } + {{- end }} + } + }, + "provider": { + {{- if eq .Provider.String "aws" }} + "aws": [{}] + {{- else if eq .Provider.String "gcp" }} + "google": [{}] + {{- else if eq .Provider.String "azure" }} + "azurerm": [{"features": [{}]}] + {{- end }} + } +} diff --git a/internal/instrumentation/buffer.go b/internal/instrumentation/buffer.go deleted file mode 100644 index 7d31922..0000000 --- a/internal/instrumentation/buffer.go +++ /dev/null @@ -1,58 +0,0 @@ -package instrumentation - -import "sync" - -// This implementation is adapted from the unexported type buffer in src/log/slog/internal/buffer/buffer.go. -type Buffer []byte - -// Having an initial size gives a dramatic speedup. -var bufPool = sync.Pool{ - New: func() any { - b := make([]byte, 0, 1024) - return (*Buffer)(&b) - }, -} - -func NewBuffer() *Buffer { - return bufPool.Get().(*Buffer) -} - -func (b *Buffer) Free() { - // To reduce peak allocation, return only smaller buffers to the pool. - const maxBufferSize = 16 << 10 - if cap(*b) <= maxBufferSize { - *b = (*b)[:0] - bufPool.Put(b) - } -} - -func (b *Buffer) Reset() { - b.SetLen(0) -} - -func (b *Buffer) Write(p []byte) (int, error) { - *b = append(*b, p...) - return len(p), nil -} - -func (b *Buffer) WriteString(s string) (int, error) { - *b = append(*b, s...) - return len(s), nil -} - -func (b *Buffer) WriteByte(c byte) error { - *b = append(*b, c) - return nil -} - -func (b *Buffer) String() string { - return string(*b) -} - -func (b *Buffer) Len() int { - return len(*b) -} - -func (b *Buffer) SetLen(n int) { - *b = (*b)[:n] -} diff --git a/internal/instrumentation/handler.go b/internal/instrumentation/handler.go deleted file mode 100644 index 8fe3613..0000000 --- a/internal/instrumentation/handler.go +++ /dev/null @@ -1,193 +0,0 @@ -package instrumentation - -import ( - "bytes" - "context" - "io" - "log/slog" - "strconv" - "strings" - "sync" -) - -const ( - timeFormat string = "2006-01-02 15:04:05 -07:00" - moduleName string = "github.com/hanzoai/o11y-foundry/" - totalLevelSpaces int = 6 // total spaces for the level key -) - -type PrettyHandler struct { - out io.Writer - opts Options - goas []groupOrAttrs - mu *sync.Mutex -} - -type Options struct { - // Level reports the minimum level to log. - // Levels with lower levels are discarded. - // If nil, the Handler uses [slog.LevelInfo]. - Level slog.Leveler - - // AddSource reports whether to add the source code location of the - // log statement to the output. - AddSource bool -} - -// groupOrAttrs holds either a group name or a list of slog.Attrs. -type groupOrAttrs struct { - group string // group name if non-empty - attrs []slog.Attr // attrs if non-empty -} - -var _ slog.Handler = (*PrettyHandler)(nil) - -func newPrettyHandler(out io.Writer, opts *Options) *PrettyHandler { - if opts == nil { - opts = &Options{ - Level: slog.LevelInfo, - AddSource: true, - } - } - - return &PrettyHandler{ - out: out, - opts: *opts, - mu: &sync.Mutex{}, - } -} - -func (handler *PrettyHandler) Enabled(ctx context.Context, level slog.Level) bool { - return level >= handler.opts.Level.Level() -} - -func (handler *PrettyHandler) Handle(ctx context.Context, record slog.Record) error { - buf := NewBuffer() - defer buf.Free() - - // write the time attribute - buf = handler.appendAttr(buf, slog.Time(slog.TimeKey, record.Time), false, false, true, '|') - - // write the level attribute - buf = handler.appendAttr(buf, slog.String(slog.LevelKey, record.Level.String()), false, true, true, '|') - - // write the source attribute - buf = handler.appendAttr(buf, slog.Any(slog.SourceKey, record.Source()), false, true, true, '-') - - // write the message attribute - buf = handler.appendAttr(buf, slog.String(slog.MessageKey, record.Message), false, true, false, 0) - - goas := handler.goas - if record.NumAttrs() == 0 { - // If the record has no Attrs, remove groups at the end of the list; they are empty. - for len(goas) > 0 && goas[len(goas)-1].group != "" { - goas = goas[:len(goas)-1] - } - } - - for _, goa := range goas { - if goa.group != "" { - buf = handler.appendAttr(buf, slog.String("group", goa.group), false, true, false, ':') - } else { - for _, attr := range goa.attrs { - buf = handler.appendAttr(buf, attr, true, true, false, 0) - } - } - } - - record.Attrs(func(attr slog.Attr) bool { - buf = handler.appendAttr(buf, attr, true, true, false, 0) - return true - }) - - _ = buf.WriteByte('\n') - - handler.mu.Lock() - defer handler.mu.Unlock() - _, err := handler.out.Write(*buf) - - return err -} - -func (handler *PrettyHandler) WithAttrs(attrs []slog.Attr) slog.Handler { - if len(attrs) == 0 { - return handler - } - - return handler.withGroupOrAttrs(groupOrAttrs{attrs: attrs}) -} - -func (handler *PrettyHandler) WithGroup(name string) slog.Handler { - if name == "" { - return handler - } - - return handler.withGroupOrAttrs(groupOrAttrs{group: name}) -} - -func (handler *PrettyHandler) withGroupOrAttrs(goa groupOrAttrs) *PrettyHandler { - copyOfHandler := *handler - copyOfHandler.goas = make([]groupOrAttrs, len(handler.goas)+1) - copy(copyOfHandler.goas, handler.goas) - copyOfHandler.goas[len(copyOfHandler.goas)-1] = goa - return ©OfHandler -} - -func (handler *PrettyHandler) appendAttr(buf *Buffer, attr slog.Attr, key bool, leftSpace bool, rightSpace bool, sep byte) *Buffer { - // Resolve the Attr's value before doing anything else. - attr.Value = attr.Value.Resolve() - - // Ignore empty Attrs. - if attr.Equal(slog.Attr{}) { - return buf - } - - // Indent 1 space if requested. - if leftSpace { - _ = buf.WriteByte(' ') - } - - // Write the attr. - switch attr.Value.Kind() { - case slog.KindTime: - if attr.Key == slog.TimeKey { - _, _ = buf.WriteString(attr.Value.Time().Format(timeFormat)) - break - } - - if key { - _, _ = buf.WriteString(attr.Key + "=") - } - _, _ = buf.WriteString(attr.Value.Time().Format(timeFormat)) - - case slog.KindAny: - if src, ok := attr.Value.Any().(*slog.Source); ok { - _, _ = buf.WriteString(strings.TrimPrefix(src.Function, moduleName) + ":" + strconv.Itoa(src.Line)) - } - default: - if key { - _, _ = buf.WriteString(attr.Key + "=") - } - - _, _ = buf.WriteString(attr.Value.String()) - - // Add spaces after the level key. - if attr.Key == slog.LevelKey { - // The total spaces should be totalLevelSpaces. - spaces := max(totalLevelSpaces-len(attr.Value.String()), 0) - _, _ = buf.Write(bytes.Repeat([]byte(" "), spaces)) - } - } - - // Add spaces after the attr. - if rightSpace { - _ = buf.WriteByte(' ') - } - - // Add a separator character. - if sep != 0 { - _ = buf.WriteByte(sep) - } - - return buf -} diff --git a/internal/instrumentation/handler_test.go b/internal/instrumentation/handler_test.go deleted file mode 100644 index 6c1057a..0000000 --- a/internal/instrumentation/handler_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package instrumentation - -import ( - "bytes" - "context" - "log/slog" - "regexp" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestPrettyHandler(t *testing.T) { - testCases := []struct { - name string - f func(logger *slog.Logger) - opts *Options - expected string - }{ - { - name: "Simple", - f: func(logger *slog.Logger) { - logger.InfoContext(context.Background(), "this is a pretty log message") - }, - opts: &Options{ - Level: slog.LevelDebug, - AddSource: true, - }, - expected: `[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [-+][0-9]{2}:[0-9]{2} \| INFO \| internal/instrumentation\.TestPrettyHandler\.func1:24 - this is a pretty log message\n`, - }, - { - name: "WithAttrs", - f: func(logger *slog.Logger) { - logger.InfoContext(context.Background(), "this is a pretty log message with attrs", slog.String("k", "v")) - }, - opts: &Options{ - Level: slog.LevelDebug, - AddSource: true, - }, - expected: `[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [-+][0-9]{2}:[0-9]{2} \| INFO \| internal/instrumentation\.TestPrettyHandler\.func2:35 - this is a pretty log message with attrs k=v\n`, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - var buf bytes.Buffer - logger := slog.New(newPrettyHandler(&buf, tc.opts)) - tc.f(logger) - - re, err := regexp.Compile(tc.expected) - require.NoError(t, err) - - assert.Regexp(t, re, buf.String()) - }) - } -} diff --git a/internal/instrumentation/logger.go b/internal/instrumentation/logger.go index bbcae96..8e5c0ad 100644 --- a/internal/instrumentation/logger.go +++ b/internal/instrumentation/logger.go @@ -5,17 +5,17 @@ import ( "os" ) +// NewLogger returns the process logger. When debug is false logs are dropped +// so the default UX matches helm/aws (silent on stdout/stderr, errors surface +// through the cobra exit path). When debug is true, debug-level records are +// emitted as JSON on stderr so stdout remains reserved for command results. func NewLogger(debug bool) *slog.Logger { - if debug { - return newLoggerWithLevel(slog.LevelDebug) + if !debug { + return slog.New(slog.DiscardHandler) } - return newLoggerWithLevel(slog.LevelInfo) -} - -func newLoggerWithLevel(level slog.Level) *slog.Logger { - return slog.New(newPrettyHandler(os.Stdout, &Options{ - AddSource: true, - Level: level, + return slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{ + Level: slog.LevelDebug, + AddSource: false, })) } diff --git a/internal/ledger/config.go b/internal/ledger/config.go new file mode 100644 index 0000000..d857181 --- /dev/null +++ b/internal/ledger/config.go @@ -0,0 +1,35 @@ +package ledger + +// key is the Segment write key, set via ldflags at build time. +// Example: -ldflags "-X github.com/signoz/foundry/internal/ledger.key=<key>". +var key string = "<unset>" + +// Config holds ledger configuration. +type Config struct { + Enabled bool + Segment Segment +} + +// Segment holds Segment-specific configuration. +type Segment struct { + Key string +} + +// NewConfig returns the default ledger configuration. +// The Segment write key is populated from the ldflags-injected value. +func NewConfig() Config { + return Config{ + Enabled: true, + Segment: Segment{ + Key: key, + }, + } +} + +// Provider returns the provider name based on the configuration. +func (c Config) Provider() string { + if c.Enabled { + return "segment" + } + return "noop" +} diff --git a/internal/ledger/ledger.go b/internal/ledger/ledger.go new file mode 100644 index 0000000..81212d8 --- /dev/null +++ b/internal/ledger/ledger.go @@ -0,0 +1,17 @@ +// Package ledger provides anonymous usage tracking for foundryctl commands. +package ledger + +import ( + "context" + + "github.com/signoz/foundry/internal/domain" +) + +// Ledger is the interface for tracking CLI usage events. +type Ledger interface { + // Track records a single foundryctl event with the given properties. + Track(ctx context.Context, event domain.Event, properties domain.Properties) + + // Close flushes any pending events and releases resources. + Close() error +} diff --git a/internal/ledger/noopledger/provider.go b/internal/ledger/noopledger/provider.go new file mode 100644 index 0000000..4841cac --- /dev/null +++ b/internal/ledger/noopledger/provider.go @@ -0,0 +1,19 @@ +package noopledger + +import ( + "context" + + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/ledger" +) + +// provider is a no-op implementation of ledger.Ledger. +type provider struct{} + +// New creates a ledger that does nothing. +func New() ledger.Ledger { + return &provider{} +} + +func (p *provider) Track(_ context.Context, _ domain.Event, _ domain.Properties) {} +func (p *provider) Close() error { return nil } diff --git a/internal/ledger/segmentledger/provider.go b/internal/ledger/segmentledger/provider.go new file mode 100644 index 0000000..deedbd9 --- /dev/null +++ b/internal/ledger/segmentledger/provider.go @@ -0,0 +1,57 @@ +package segmentledger + +import ( + "context" + "fmt" + "runtime" + + segment "github.com/segmentio/analytics-go/v3" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/ledger" + "github.com/signoz/foundry/internal/ledger/noopledger" + "github.com/signoz/foundry/internal/version" +) + +// provider implements ledger.Ledger using Segment. +type provider struct { + client segment.Client +} + +// New creates a new Segment ledger provider. +// Returns a noop provider if the write key is not set. +func New(config ledger.Config) ledger.Ledger { + if config.Segment.Key == "" || config.Segment.Key == "<unset>" { + return noopledger.New() + } + + client, err := segment.NewWithConfig(config.Segment.Key, segment.Config{}) + if err != nil { + return noopledger.New() + } + + return &provider{ + client: client, + } +} + +func (p *provider) Track(_ context.Context, event domain.Event, properties domain.Properties) { + properties = properties. + Set("os", runtime.GOOS). + Set("arch", runtime.GOARCH). + Set("foundry_version", version.Info.Version()) + + props := segment.NewProperties() + for k, v := range properties.Map() { + props.Set(k, v) + } + + _ = p.client.Enqueue(segment.Track{ + AnonymousId: domain.MustNewDistinctID().String(), + Event: fmt.Sprintf("foundryctl: %s", event.String()), + Properties: props, + }) +} + +func (p *provider) Close() error { + return p.client.Close() +} diff --git a/internal/molding/collectionagent/collectormolding/collector.go b/internal/molding/collectionagent/collectormolding/collector.go new file mode 100644 index 0000000..877e848 --- /dev/null +++ b/internal/molding/collectionagent/collectormolding/collector.go @@ -0,0 +1,33 @@ +package collectormolding + +import ( + "log/slog" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/collectionagent" + collectionagentmolding "github.com/signoz/foundry/internal/molding/collectionagent" + + "context" +) + +var _ collectionagentmolding.Molding = (*collector)(nil) + +type collector struct { + logger *slog.Logger +} + +func New(logger *slog.Logger) *collector { + return &collector{logger: logger} +} + +func (m *collector) Kind() v1alpha1.MoldingKind { + return v1alpha1.MoldingKindCollector +} + +func (m *collector) MoldV1Alpha1(ctx context.Context, config *collectionagent.Casting) error { + if config.Spec.Collector.Status.Env == nil { + config.Spec.Collector.Status.Env = make(map[string]string) + } + config.Spec.Collector.Status.Env["OTEL_COLLECTOR_KIND"] = config.Spec.Collector.Kind.String() + return nil +} diff --git a/internal/molding/collectionagent/molding.go b/internal/molding/collectionagent/molding.go new file mode 100644 index 0000000..2433a54 --- /dev/null +++ b/internal/molding/collectionagent/molding.go @@ -0,0 +1,17 @@ +package collectionagent + +import ( + "context" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/collectionagent" +) + +type MoldingEnricher interface { + EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *collectionagent.Casting) error +} + +type Molding interface { + Kind() v1alpha1.MoldingKind + MoldV1Alpha1(ctx context.Context, config *collectionagent.Casting) error +} diff --git a/internal/molding/ingestermolding/ingester.go b/internal/molding/ingestermolding/ingester.go index 6572824..f14ffc2 100644 --- a/internal/molding/ingestermolding/ingester.go +++ b/internal/molding/ingestermolding/ingester.go @@ -3,9 +3,7 @@ package ingestermolding import ( "bytes" "context" - "fmt" "log/slog" - "strings" "github.com/hanzoai/o11y-foundry/api/v1alpha1" foundryerrors "github.com/hanzoai/o11y-foundry/internal/errors" @@ -28,7 +26,7 @@ func (molding *ingester) Kind() v1alpha1.MoldingKind { return v1alpha1.MoldingKindIngester } -func (molding *ingester) MoldV1Alpha1(ctx context.Context, config *v1alpha1.Casting) error { +func (molding *ingester) MoldV1Alpha1(ctx context.Context, config *installation.Casting) error { // render the template for config.yaml data, err := molding.getData(config) if err != nil { @@ -51,6 +49,11 @@ func (molding *ingester) MoldV1Alpha1(ctx context.Context, config *v1alpha1.Cast "opamp.yaml": opampBuf.String(), } + if config.Spec.Ingester.Status.Env == nil { + config.Spec.Ingester.Status.Env = make(map[string]string) + } + config.Spec.Ingester.Status.Env["SIGNOZ_OTEL_COLLECTOR_TIMEOUT"] = "10m" + return nil } @@ -62,7 +65,7 @@ func (molding *ingester) getData(config *v1alpha1.Casting) (Data, error) { o11yAddress := config.Spec.O11y.Status.Addresses.Opamp[0] if len(config.Spec.TelemetryStore.Status.Addresses.TCP) == 0 { - return Data{}, fmt.Errorf("telemetry store address is not set") + return Data{}, foundryerrors.Newf(foundryerrors.TypeInternal, "telemetry store address is not set") } telemetryStoreAddresses := config.Spec.TelemetryStore.Status.Addresses.TCP diff --git a/internal/molding/ingestermolding/template.go b/internal/molding/ingestermolding/template.go index db3c605..bbe5518 100644 --- a/internal/molding/ingestermolding/template.go +++ b/internal/molding/ingestermolding/template.go @@ -10,8 +10,8 @@ import ( var templates embed.FS var ( - ConfigV0129xTemplate *types.Template = types.MustNewTemplateFromFS(templates, "templates/config.v0129x.yaml.gotmpl", types.FormatYAML) - OpampV0129xTemplate *types.Template = types.MustNewTemplateFromFS(templates, "templates/opamp.v0129x.yaml.gotmpl", types.FormatYAML) + ConfigV0129xTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/config.v0129x.yaml.gotmpl", domain.FormatYAML) + OpampV0129xTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/opamp.v0129x.yaml.gotmpl", domain.FormatYAML) ) type Data struct { diff --git a/internal/molding/ingestermolding/templates/config.v0129x.yaml.gotmpl b/internal/molding/ingestermolding/templates/config.v0129x.yaml.gotmpl index 5e7da0e..728061a 100644 --- a/internal/molding/ingestermolding/templates/config.v0129x.yaml.gotmpl +++ b/internal/molding/ingestermolding/templates/config.v0129x.yaml.gotmpl @@ -14,9 +14,9 @@ receivers: endpoint: "0.0.0.0:4318" processors: batch: - send_batch_size: 20000 - send_batch_max_size: 25000 - timeout: 1s + send_batch_size: 50000 + send_batch_max_size: 55000 + timeout: 5s batch/meter: send_batch_size: 20000 send_batch_max_size: 25000 @@ -76,6 +76,12 @@ exporters: timeout: 45s sending_queue: enabled: false + metadataexporter: + enabled: true + dsn: {{ .TelemetryStoreMetadataAddress }} + timeout: 45s + cache: + provider: in_memory extensions: o11y_health_check: endpoint: "0.0.0.0:13133" diff --git a/internal/molding/metastoremolding/metastore.go b/internal/molding/metastoremolding/metastore.go index a019d26..41378a7 100644 --- a/internal/molding/metastoremolding/metastore.go +++ b/internal/molding/metastoremolding/metastore.go @@ -21,7 +21,7 @@ func (molding *metastore) Kind() v1alpha1.MoldingKind { return v1alpha1.MoldingKindMetaStore } -func (molding *metastore) MoldV1Alpha1(ctx context.Context, config *v1alpha1.Casting) error { +func (molding *metastore) MoldV1Alpha1(ctx context.Context, config *installation.Casting) error { if config.Spec.MetaStore.Status.Env == nil { config.Spec.MetaStore.Status.Env = make(map[string]string) } @@ -31,7 +31,12 @@ func (molding *metastore) MoldV1Alpha1(ctx context.Context, config *v1alpha1.Cas } switch config.Spec.MetaStore.Kind { - case v1alpha1.MetaStoreKindPostgres: + case installation.MetaStoreKindSQLite: + replicas := config.Spec.MetaStore.Spec.Cluster.Replicas + if replicas != nil && *replicas != 1 { + return errors.Newf(errors.TypeInvalidInput, "metastore.spec.cluster.replicas must be 1 when metastore.kind is sqlite; sqlite is embedded and per-instance storage is driven by signoz.spec.cluster.replicas (got %d)", *replicas) + } + case installation.MetaStoreKindPostgres: if val, ok := config.Spec.MetaStore.Spec.Env["POSTGRES_DB"]; ok { molding.logger.WarnContext(ctx, "POSTGRES_DB is going to be overridden", slog.String("value", val)) } diff --git a/internal/molding/molding.go b/internal/molding/molding.go index 872f23e..e897ab6 100644 --- a/internal/molding/molding.go +++ b/internal/molding/molding.go @@ -6,13 +6,16 @@ import ( "github.com/hanzoai/o11y-foundry/api/v1alpha1" ) +// MoldingEnricher populates a molding's Status fields from the surrounding +// installation casting. The ordering of EnrichStatus calls is owned by the +// installation Planner, which iterates the kinds it knows about. type MoldingEnricher interface { - // Enrich the molding status with the casting configuration. - EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *v1alpha1.Casting) error + EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *installation.Casting) error } +// Molding generates materials for a single SigNoz component. Mutates the +// config in place; not safe for concurrent use. type Molding interface { - // Kind of the molding. Kind() v1alpha1.MoldingKind // Molds the v1alpha1 casting configuration. This function mutates the config in place. It is not safe for concurrent use. diff --git a/internal/molding/signozmolding/signoz.go b/internal/molding/signozmolding/signoz.go new file mode 100644 index 0000000..74cf567 --- /dev/null +++ b/internal/molding/signozmolding/signoz.go @@ -0,0 +1,81 @@ +package signozmolding + +import ( + "context" + "fmt" + "log/slog" + "strings" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/installation" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/molding" +) + +var _ molding.Molding = (*signoz)(nil) + +type signoz struct { + logger *slog.Logger +} + +func New(logger *slog.Logger) *signoz { + return &signoz{ + logger: logger, + } +} + +func (molding *signoz) Kind() v1alpha1.MoldingKind { + return v1alpha1.MoldingKindSignoz +} + +func (molding *signoz) MoldV1Alpha1(ctx context.Context, config *installation.Casting) error { + if config.Spec.Signoz.Status.Env == nil { + config.Spec.Signoz.Status.Env = make(map[string]string) + } + + if config.Spec.Signoz.Spec.Env == nil { + config.Spec.Signoz.Spec.Env = make(map[string]string) + } + + // Add telemetry store addresses + config.Spec.Signoz.Status.Env["SIGNOZ_TELEMETRYSTORE_PROVIDER"] = config.Spec.TelemetryStore.Kind.String() + + if val, ok := config.Spec.Signoz.Spec.Env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN"]; ok { + molding.logger.WarnContext(ctx, "SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN is going to be overridden", slog.String("value", val)) + } + + if len(config.Spec.TelemetryStore.Status.Addresses.TCP) == 0 { + return errors.Newf(errors.TypeInternal, "telemetry store address is not set") + } + config.Spec.Signoz.Status.Env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN"] = config.Spec.TelemetryStore.Status.Addresses.TCP[0] + + // Add metastore addresses + config.Spec.Signoz.Status.Env["SIGNOZ_SQLSTORE_PROVIDER"] = config.Spec.MetaStore.Kind.String() + + switch config.Spec.MetaStore.Kind { + case installation.MetaStoreKindSQLite: + config.Spec.Signoz.Status.Env["SIGNOZ_SQLSTORE_SQLITE_PATH"] = "/var/lib/signoz/signoz.db" + case installation.MetaStoreKindPostgres: + if config.Spec.MetaStore.Status.Addresses.DSN != nil { + if val, ok := config.Spec.Signoz.Spec.Env["SIGNOZ_SQLSTORE_POSTGRES_DSN"]; ok { + molding.logger.WarnContext(ctx, "SIGNOZ_SQLSTORE_POSTGRES_DSN is going to be overridden", slog.String("value", val)) + } + // construct postgres dsn with user, password, host, port, and db + addrs, err := domain.ParseAddresses(config.Spec.MetaStore.Status.Addresses.DSN) + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to parse addresses") + } + var dsns []string + user := config.Spec.MetaStore.Status.Env["POSTGRES_USER"] + password := config.Spec.MetaStore.Status.Env["POSTGRES_PASSWORD"] + db := config.Spec.MetaStore.Status.Env["POSTGRES_DB"] + for _, addr := range addrs { + dsn := fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=disable", user, password, addr.Host(), addr.Port(), db) + dsns = append(dsns, dsn) + } + config.Spec.Signoz.Status.Env["SIGNOZ_SQLSTORE_POSTGRES_DSN"] = strings.Join(dsns, ",") + } + } + return nil +} diff --git a/internal/molding/telemetrykeepermolding/telemetrykeeper.go b/internal/molding/telemetrykeepermolding/telemetrykeeper.go index 9e92435..36db870 100644 --- a/internal/molding/telemetrykeepermolding/telemetrykeeper.go +++ b/internal/molding/telemetrykeepermolding/telemetrykeeper.go @@ -27,22 +27,37 @@ func (molding *telemetrykeeper) Kind() v1alpha1.MoldingKind { return v1alpha1.MoldingKindTelemetryKeeper } -func (molding *telemetrykeeper) MoldV1Alpha1(ctx context.Context, config *v1alpha1.Casting) error { +func (molding *telemetrykeeper) MoldV1Alpha1(ctx context.Context, config *installation.Casting) error { data, err := newData(config) if err != nil { molding.logger.ErrorContext(ctx, "failed to get data", foundryerrors.LogAttr(err)) return err } + // Extract enricher config overrides (applies to all keeper nodes). + overrides := config.Spec.TelemetryKeeper.Status.Extras["_overrides"] + // Generate per-server configs (each keeper node needs its own server_id) configs := make(map[string]string, data.ServerCount) for i := 0; i < data.ServerCount; i++ { configBuf := bytes.NewBuffer(nil) data.ServerID = i // 0-indexed, used for array indexing in template if err := KeeperClickhousev2556YAML.Execute(configBuf, data); err != nil { - return fmt.Errorf("failed to execute keeper template for server %d: %w", data.ServerID, err) + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to execute keeper template for server %d", data.ServerID) } - configs[fmt.Sprintf("keeper-%d.yaml", i)] = configBuf.String() + + key := fmt.Sprintf("keeper-%d.yaml", i) + base := configBuf.String() + + if overrides != "" { + merged, err := domain.MergeYAML(base, overrides) + if err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to merge config overrides for %s", key) + } + base = merged + } + + configs[key] = base } config.Spec.TelemetryKeeper.Status.Config.Data = configs diff --git a/internal/molding/telemetrykeepermolding/template.go b/internal/molding/telemetrykeepermolding/template.go index 5e31270..cfe0d66 100644 --- a/internal/molding/telemetrykeepermolding/template.go +++ b/internal/molding/telemetrykeepermolding/template.go @@ -2,7 +2,6 @@ package telemetrykeepermolding import ( "embed" - "fmt" "github.com/hanzoai/o11y-foundry/api/v1alpha1" "github.com/hanzoai/o11y-foundry/internal/types" @@ -12,18 +11,18 @@ import ( var templates embed.FS var ( - KeeperClickhousev2556YAML *types.Template = types.MustNewTemplateFromFS(templates, "templates/keeper.clickhouse.v2556.yaml.gotmpl", types.FormatYAML) + KeeperClickhousev2556YAML *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/keeper.clickhouse.v2556.yaml.gotmpl", domain.FormatYAML) ) // Data is the template data for rendering ClickHouse Keeper configs. type Data struct { - RaftAddresses []types.Address // Inter-keeper consensus addresses - ClientAddresses []types.Address // Client-facing addresses + RaftAddresses []domain.Address // Inter-keeper consensus addresses + ClientAddresses []domain.Address // Client-facing addresses ServerCount int ServerID int // Current server ID for per-node config generation } -func newData(config *v1alpha1.Casting) (Data, error) { +func newData(config *installation.Casting) (Data, error) { var data Data if config.Spec.TelemetryKeeper.Spec.Cluster.Replicas == nil { @@ -34,23 +33,23 @@ func newData(config *v1alpha1.Casting) (Data, error) { raftAddresses := config.Spec.TelemetryKeeper.Status.Addresses.Raft if len(raftAddresses) < data.ServerCount { - return Data{}, fmt.Errorf("insufficient raft addresses: have %d, need %d servers", len(raftAddresses), data.ServerCount) + return Data{}, errors.Newf(errors.TypeInvalidInput, "insufficient raft addresses: have %d, need %d servers", len(raftAddresses), data.ServerCount) } clientAddresses := config.Spec.TelemetryKeeper.Status.Addresses.Client if len(clientAddresses) < data.ServerCount { - return Data{}, fmt.Errorf("insufficient client addresses: have %d, need %d servers", len(clientAddresses), data.ServerCount) + return Data{}, errors.Newf(errors.TypeInvalidInput, "insufficient client addresses: have %d, need %d servers", len(clientAddresses), data.ServerCount) } - newRaftAddrs, err := types.NewAddresses(raftAddresses[:data.ServerCount]) + newRaftAddrs, err := domain.ParseAddresses(raftAddresses[:data.ServerCount]) if err != nil { - return Data{}, fmt.Errorf("failed to parse raft addresses: %w", err) + return Data{}, errors.Wrapf(err, errors.TypeInternal, "failed to parse raft addresses") } data.RaftAddresses = newRaftAddrs - newClientAddrs, err := types.NewAddresses(clientAddresses[:data.ServerCount]) + newClientAddrs, err := domain.ParseAddresses(clientAddresses[:data.ServerCount]) if err != nil { - return Data{}, fmt.Errorf("failed to parse client addresses: %w", err) + return Data{}, errors.Wrapf(err, errors.TypeInternal, "failed to parse client addresses") } data.ClientAddresses = newClientAddrs diff --git a/internal/molding/telemetrykeepermolding/templates/keeper.clickhouse.v2556.yaml.gotmpl b/internal/molding/telemetrykeepermolding/templates/keeper.clickhouse.v2556.yaml.gotmpl index aa6417c..a20ab9a 100644 --- a/internal/molding/telemetrykeepermolding/templates/keeper.clickhouse.v2556.yaml.gotmpl +++ b/internal/molding/telemetrykeepermolding/templates/keeper.clickhouse.v2556.yaml.gotmpl @@ -8,6 +8,9 @@ keeper_server: operation_timeout_ms: 10000 raft_logs_level: warning session_timeout_ms: 30000 + force_sync: false + snapshot_distance: 100000 + snapshots_to_keep: 3 log_storage_path: /var/lib/clickhouse/coordination/log raft_configuration: server: diff --git a/internal/molding/telemetrystoremolding/telemetrystore.go b/internal/molding/telemetrystoremolding/telemetrystore.go index a08fd76..c7379ed 100644 --- a/internal/molding/telemetrystoremolding/telemetrystore.go +++ b/internal/molding/telemetrystoremolding/telemetrystore.go @@ -28,62 +28,94 @@ func (molding *telemetrystore) Kind() v1alpha1.MoldingKind { return v1alpha1.MoldingKindTelemetryStore } -func (molding *telemetrystore) MoldV1Alpha1(ctx context.Context, config *v1alpha1.Casting) error { +func (molding *telemetrystore) MoldV1Alpha1(ctx context.Context, config *installation.Casting) error { data, err := molding.getData(config) if err != nil { molding.logger.ErrorContext(ctx, "failed to get data", foundryerrors.LogAttr(err)) return err } - configBuf := bytes.NewBuffer(nil) - if err := ConfigClickhousev2556YAML.Execute(configBuf, data); err != nil { - return fmt.Errorf("failed to execute config template: %w", err) - } + // Extract enricher config overrides (applies to all nodes). + overrides := config.Spec.TelemetryStore.Status.Extras["_overrides"] functionBuf := bytes.NewBuffer(nil) if err := FunctionsClickhousev2556YAML.Execute(functionBuf, data); err != nil { - return fmt.Errorf("failed to execute config template: %w", err) + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to execute functions template") } - config.Spec.TelemetryStore.Status.Config.Data = map[string]string{ - "config.yaml": configBuf.String(), - "functions.yaml": functionBuf.String(), + + // Generate per-node configs (each node needs its own macros.shard / macros.replica). + configs := make(map[string]string, data.ShardCount*data.ReplicaCount+1) + for s := 0; s < data.ShardCount; s++ { + for r := 0; r < data.ReplicaCount; r++ { + data.ShardID = s + data.ReplicaID = r + + configBuf := bytes.NewBuffer(nil) + if err := ConfigClickhousev2556YAML.Execute(configBuf, data); err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to execute config template for shard %d replica %d", s, r) + } + + key := fmt.Sprintf("config-%d-%d.yaml", s, r) + base := configBuf.String() + + if overrides != "" { + merged, err := domain.MergeYAML(base, overrides) + if err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to merge config overrides for %s", key) + } + base = merged + } + + configs[key] = base + } } + configs["functions.yaml"] = functionBuf.String() + + config.Spec.TelemetryStore.Status.Config.Data = configs return nil } -func (molding *telemetrystore) getData(config *v1alpha1.Casting) (Data, error) { +func (molding *telemetrystore) getData(config *installation.Casting) (Data, error) { storeAddresses := config.Spec.TelemetryStore.Status.Addresses.TCP if len(storeAddresses) == 0 { - return Data{}, fmt.Errorf("telemetry store addresses not set in status") + return Data{}, foundryerrors.Newf(foundryerrors.TypeInternal, "telemetry store addresses not set in status") } cluster := config.Spec.TelemetryStore.Spec.Cluster - shardCount := max(*cluster.Shards, 1) - replicaCount := max(*cluster.Replicas, 1) + shardCount := 1 + if cluster.Shards != nil && *cluster.Shards > 0 { + shardCount = *cluster.Shards + } + + replicaCount := 1 + if cluster.Replicas != nil { + replicaCount = *cluster.Replicas + 1 + } expectedNodes := shardCount * replicaCount if len(storeAddresses) < expectedNodes { - return Data{}, fmt.Errorf( + return Data{}, foundryerrors.Newf( + foundryerrors.TypeInvalidInput, "insufficient addresses: have %d, need %d (shards=%d × replicas=%d)", len(storeAddresses), expectedNodes, shardCount, replicaCount, ) } - newStoreAddrs, err := types.NewAddresses(storeAddresses[:expectedNodes]) + newStoreAddrs, err := domain.ParseAddresses(storeAddresses[:expectedNodes]) if err != nil { - return Data{}, fmt.Errorf("failed to parse addresses: %w", err) + return Data{}, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to parse addresses") } keeperAddresses := config.Spec.TelemetryKeeper.Status.Addresses.Client if len(keeperAddresses) == 0 { - return Data{}, fmt.Errorf("telemetry keeper addresses not set in status") + return Data{}, foundryerrors.Newf(foundryerrors.TypeInternal, "telemetry keeper addresses not set in status") } - newKeeperAddrs, err := types.NewAddresses(keeperAddresses) + newKeeperAddrs, err := domain.ParseAddresses(keeperAddresses) if err != nil { - return Data{}, fmt.Errorf("failed to parse addresses: %w", err) + return Data{}, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to parse addresses") } return Data{ diff --git a/internal/molding/telemetrystoremolding/template.go b/internal/molding/telemetrystoremolding/template.go index dcc24e4..472b1ce 100644 --- a/internal/molding/telemetrystoremolding/template.go +++ b/internal/molding/telemetrystoremolding/template.go @@ -10,14 +10,16 @@ import ( var templates embed.FS var ( - ConfigClickhousev2556YAML *types.Template = types.MustNewTemplateFromFS(templates, "templates/config.clickhouse.v2556.yaml.gotmpl", types.FormatYAML) - FunctionsClickhousev2556YAML *types.Template = types.MustNewTemplateFromFS(templates, "templates/functions.clickhouse.v2556.yaml.gotmpl", types.FormatYAML) + ConfigClickhousev2556YAML *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/config.clickhouse.v2556.yaml.gotmpl", domain.FormatYAML) + FunctionsClickhousev2556YAML *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/functions.clickhouse.v2556.yaml.gotmpl", domain.FormatYAML) ) // Data is the template data for rendering ClickHouse telemetry store configs. type Data struct { - StoreAddresses []types.Address - KeeperAddresses []types.Address + StoreAddresses []domain.Address + KeeperAddresses []domain.Address ShardCount int ReplicaCount int + ShardID int // 0-indexed, used to render per-node macros.shard + ReplicaID int // 0-indexed, used to render per-node macros.replica } diff --git a/internal/molding/telemetrystoremolding/templates/config.clickhouse.v2556.yaml.gotmpl b/internal/molding/telemetrystoremolding/templates/config.clickhouse.v2556.yaml.gotmpl index 3fc5959..9b485f8 100644 --- a/internal/molding/telemetrystoremolding/templates/config.clickhouse.v2556.yaml.gotmpl +++ b/internal/molding/telemetrystoremolding/templates/config.clickhouse.v2556.yaml.gotmpl @@ -17,13 +17,13 @@ logger: level: information size: 1000M macros: - replica: "01" - shard: "01" + replica: "{{ printf "%02d" .ReplicaID }}" + shard: "{{ printf "%02d" .ShardID }}" profiles: default: + allow_simdjson: 0 load_balancing: random log_queries: 1 - max_memory_usage: 10000000000 quotas: default: interval: @@ -40,13 +40,13 @@ remote_servers: cluster: shard: {{- range $s := until .ShardCount }} - - replica: - {{- range $r := until $.ReplicaCount }} + - replica: + {{- range $r := until $.ReplicaCount }} {{- $idx := add (mul $s $.ReplicaCount) $r }} {{- $addr := index $.StoreAddresses $idx }} - host: {{ $addr.Host }} - port: {{ $addr.Port }} - {{- end }} + - host: {{ $addr.Host }} + port: {{ $addr.Port }} +{{- end }} {{- end }} zookeeper: node: @@ -54,6 +54,40 @@ zookeeper: - host: {{ $addr.Host }} port: {{ $addr.Port }} {{- end }} +query_log: + flush_interval_milliseconds: 30000 + partition_by: toYYYYMM(event_date) + ttl: "event_date + INTERVAL 1 DAY DELETE" +query_thread_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" +query_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" +query_views_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" +part_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" +metric_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" +asynchronous_metric_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" +trace_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" +error_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" +latency_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" +processors_profile_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" +session_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" +text_log: + flush_interval_milliseconds: 30000 + ttl: "event_date + INTERVAL 1 DAY DELETE" +zookeeper_log: + ttl: "event_date + INTERVAL 1 DAY DELETE" tcp_port: 9000 user_defined_executable_functions_config: '*function.yaml' user_scripts_path: /var/lib/clickhouse/user_scripts/ diff --git a/internal/patch/jsonpatch/patch.go b/internal/patch/jsonpatch/patch.go new file mode 100644 index 0000000..37d5bb7 --- /dev/null +++ b/internal/patch/jsonpatch/patch.go @@ -0,0 +1,77 @@ +package jsonpatch + +import ( + "context" + "encoding/json" + + jsonpatchv5 "github.com/evanphx/json-patch/v5" + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/patch" +) + +var _ patch.Patch = (*jsonPatch)(nil) + +type jsonPatch struct{} + +func New() patch.Patch { + return &jsonPatch{} +} + +func (p *jsonPatch) Apply(ctx context.Context, materials []domain.Material, pe v1alpha1.PatchEntry) ([]domain.Material, error) { + patchDoc, err := json.Marshal(pe.Operations) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to marshal patch operations for target %q", pe.Target) + } + + result := make([]domain.Material, len(materials)) + copy(result, materials) + + matched := false + for i, mat := range result { + ok, err := patch.MatchTarget(pe.Target, mat.Path()) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInvalidInput, "invalid glob pattern %q", pe.Target) + } + if !ok { + continue + } + + structured, ok := mat.(domain.StructuredMaterial) + if !ok { + return nil, errors.Newf(errors.TypeUnsupported, "json patch on blob material %q is not supported", mat.Path()) + } + + if structured.HasMultipleDocuments() { + return nil, errors.Newf(errors.TypeUnsupported, "json patch on multi-doc yaml material %q is not supported", mat.Path()) + } + + matched = true + patched, err := applyToMaterial(structured, patchDoc) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to apply patch to %q", mat.Path()) + } + result[i] = patched + } + + if !matched { + return nil, errors.Newf(errors.TypeNotFound, "patch target %q did not match any generated material", pe.Target) + } + + return result, nil +} + +func applyToMaterial(mat domain.StructuredMaterial, patchDoc []byte) (domain.StructuredMaterial, error) { + decoded, err := jsonpatchv5.DecodePatch(patchDoc) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInvalidInput, "failed to decode json patch") + } + + patched, err := decoded.Apply(mat.JSONContents()) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to apply json patch") + } + + return mat.CloneWithJSONContents(patched), nil +} diff --git a/internal/patch/jsonpatch/patch_test.go b/internal/patch/jsonpatch/patch_test.go new file mode 100644 index 0000000..774b999 --- /dev/null +++ b/internal/patch/jsonpatch/patch_test.go @@ -0,0 +1,219 @@ +package jsonpatch + +import ( + "context" + "testing" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/internal/domain" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newYAMLMaterial(t *testing.T, yamlContent string, path string) domain.Material { + t.Helper() + mat, err := domain.NewYAMLMaterial([]byte(yamlContent), path) + require.NoError(t, err) + return mat +} + +func structuredMaterial(t *testing.T, material domain.Material) domain.StructuredMaterial { + t.Helper() + structured, ok := material.(domain.StructuredMaterial) + require.True(t, ok, "expected structured material") + return structured +} + +func TestApply_ReplaceOperation(t *testing.T) { + p := New() + mat := newYAMLMaterial(t, ` +services: + clickhouse: + mem_limit: "2G" +`, "docker-compose.yaml") + + pe := v1alpha1.PatchEntry{ + Target: "docker-compose.yaml", + Operations: []v1alpha1.PatchOperation{ + {Op: "replace", Path: "/services/clickhouse/mem_limit", Value: "4G"}, + }, + } + + result, err := p.Apply(context.Background(), []domain.Material{mat}, pe) + require.NoError(t, err) + require.Len(t, result, 1) + + contents := structuredMaterial(t, result[0]).FmtContents() + assert.Contains(t, string(contents), "4G") + assert.NotContains(t, string(contents), "2G") +} + +func TestApply_AddOperation(t *testing.T) { + p := New() + mat := newYAMLMaterial(t, ` +services: + signoz: + environment: + - "EXISTING=true" +`, "docker-compose.yaml") + + pe := v1alpha1.PatchEntry{ + Target: "docker-compose.yaml", + Operations: []v1alpha1.PatchOperation{ + {Op: "add", Path: "/services/signoz/environment/-", Value: "CUSTOM_VAR=value"}, + }, + } + + result, err := p.Apply(context.Background(), []domain.Material{mat}, pe) + require.NoError(t, err) + + contents := structuredMaterial(t, result[0]).FmtContents() + assert.Contains(t, string(contents), "CUSTOM_VAR=value") + assert.Contains(t, string(contents), "EXISTING=true") +} + +func TestApply_RemoveOperation(t *testing.T) { + p := New() + mat := newYAMLMaterial(t, ` +services: + clickhouse: + mem_limit: "2G" + cpu_count: 4 +`, "docker-compose.yaml") + + pe := v1alpha1.PatchEntry{ + Target: "docker-compose.yaml", + Operations: []v1alpha1.PatchOperation{ + {Op: "remove", Path: "/services/clickhouse/cpu_count"}, + }, + } + + result, err := p.Apply(context.Background(), []domain.Material{mat}, pe) + require.NoError(t, err) + + contents := structuredMaterial(t, result[0]).FmtContents() + assert.NotContains(t, string(contents), "cpu_count") + assert.Contains(t, string(contents), "mem_limit") +} + +func TestApply_GlobTarget(t *testing.T) { + p := New() + mat1 := newYAMLMaterial(t, `replicas: 1`, "clickhouse-shard-0.yaml") + mat2 := newYAMLMaterial(t, `replicas: 1`, "clickhouse-shard-1.yaml") + mat3 := newYAMLMaterial(t, `replicas: 1`, "signoz.yaml") + + pe := v1alpha1.PatchEntry{ + Target: "clickhouse-shard-*.yaml", + Operations: []v1alpha1.PatchOperation{ + {Op: "replace", Path: "/replicas", Value: 3}, + }, + } + + result, err := p.Apply(context.Background(), []domain.Material{mat1, mat2, mat3}, pe) + require.NoError(t, err) + require.Len(t, result, 3) + + for _, i := range []int{0, 1} { + contents := structuredMaterial(t, result[i]).FmtContents() + assert.Contains(t, string(contents), "3") + } + + contents := structuredMaterial(t, result[2]).FmtContents() + assert.Contains(t, string(contents), "1") +} + +func TestApply_FullPathMatch(t *testing.T) { + p := New() + mat := newYAMLMaterial(t, `name: test`, "deployment/compose.yaml") + + pe := v1alpha1.PatchEntry{ + Target: "deployment/compose.yaml", + Operations: []v1alpha1.PatchOperation{ + {Op: "replace", Path: "/name", Value: "patched"}, + }, + } + + result, err := p.Apply(context.Background(), []domain.Material{mat}, pe) + require.NoError(t, err) + + contents := structuredMaterial(t, result[0]).FmtContents() + assert.Contains(t, string(contents), "patched") +} + +func TestApply_UnmatchedTargetReturnsError(t *testing.T) { + p := New() + mat := newYAMLMaterial(t, `name: test`, "test.yaml") + + pe := v1alpha1.PatchEntry{ + Target: "nonexistent.yaml", + Operations: []v1alpha1.PatchOperation{ + {Op: "replace", Path: "/name", Value: "patched"}, + }, + } + + _, err := p.Apply(context.Background(), []domain.Material{mat}, pe) + assert.Error(t, err) + assert.Contains(t, err.Error(), "did not match any generated material") +} + +func TestApply_InvalidPathReturnsError(t *testing.T) { + p := New() + mat := newYAMLMaterial(t, `name: test`, "test.yaml") + + pe := v1alpha1.PatchEntry{ + Target: "test.yaml", + Operations: []v1alpha1.PatchOperation{ + {Op: "replace", Path: "/nonexistent/path", Value: "value"}, + }, + } + + _, err := p.Apply(context.Background(), []domain.Material{mat}, pe) + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to apply") +} + +func TestApply_BlobMaterialReturnsError(t *testing.T) { + p := New() + mat := domain.NewBlobMaterial([]byte("FROM alpine\n"), "Dockerfile") + + pe := v1alpha1.PatchEntry{ + Target: "Dockerfile", + Operations: []v1alpha1.PatchOperation{ + {Op: "replace", Path: "/from", Value: "ubuntu"}, + }, + } + + _, err := p.Apply(context.Background(), []domain.Material{mat}, pe) + require.Error(t, err) + assert.Contains(t, err.Error(), "json patch on blob material") +} + +func TestApply_StructuredValue(t *testing.T) { + p := New() + mat := newYAMLMaterial(t, ` +spec: + template: + spec: + containers: [] +`, "deployment.yaml") + + pe := v1alpha1.PatchEntry{ + Target: "deployment.yaml", + Operations: []v1alpha1.PatchOperation{ + { + Op: "add", + Path: "/spec/template/spec/tolerations", + Value: []map[string]string{ + {"key": "dedicated", "value": "signoz", "effect": "NoSchedule"}, + }, + }, + }, + } + + result, err := p.Apply(context.Background(), []domain.Material{mat}, pe) + require.NoError(t, err) + + contents := structuredMaterial(t, result[0]).FmtContents() + assert.Contains(t, string(contents), "dedicated") + assert.Contains(t, string(contents), "NoSchedule") +} diff --git a/internal/patch/patch.go b/internal/patch/patch.go new file mode 100644 index 0000000..79f4518 --- /dev/null +++ b/internal/patch/patch.go @@ -0,0 +1,21 @@ +package patch + +import ( + "context" + "path/filepath" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/internal/domain" +) + +// Patch applies a single patch entry to generated materials. +type Patch interface { + // Apply applies a single patch entry to matching materials and returns the patched materials. + Apply(ctx context.Context, materials []domain.Material, patch v1alpha1.PatchEntry) ([]domain.Material, error) +} + +// MatchTarget checks if a material path matches a target pattern. +// Supports exact paths and glob patterns. +func MatchTarget(pattern, path string) (bool, error) { + return filepath.Match(pattern, path) +} diff --git a/internal/patch/patch_test.go b/internal/patch/patch_test.go new file mode 100644 index 0000000..a37ab35 --- /dev/null +++ b/internal/patch/patch_test.go @@ -0,0 +1,34 @@ +package patch + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMatchTarget(t *testing.T) { + tests := []struct { + name string + pattern string + path string + want bool + }{ + {"exact match", "test.yaml", "test.yaml", true}, + {"full path match", "deployment/compose.yaml", "deployment/compose.yaml", true}, + {"basename does not match full path", "compose.yaml", "deployment/compose.yaml", false}, + {"glob match", "*.yaml", "test.yaml", true}, + {"glob no match", "*.json", "test.yaml", false}, + {"prefix glob", "clickhouse-*.yaml", "clickhouse-shard-0.yaml", true}, + {"path glob", "deployment/*.yaml", "deployment/compose.yaml", true}, + {"no match", "other.yaml", "test.yaml", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := MatchTarget(tt.pattern, tt.path) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/internal/planner/doc.go b/internal/planner/doc.go new file mode 100644 index 0000000..c8f0353 --- /dev/null +++ b/internal/planner/doc.go @@ -0,0 +1,19 @@ +// Package planner defines the per-Kind orchestrator contract that Foundry +// iterates against. +// +// Each Kind (Installation, CollectionAgent, ...) ships a Planner implementation +// in its own casting package. Foundry resolves a Kind to a Planner at runtime +// via the dispatch map on the Foundry struct and then drives the same set of +// verbs (Machinery, Patches, Toolers, MoldingKinds, EnrichStatus, Mold, +// MergeStatusIntoSpec, Forge, Cast) regardless of which Kind it is. +// +// Keep this package free of orchestration and side effects. It must not +// construct planners, decide which Kind runs, dispatch by Kind, execute +// tools, write files, or hold registries. The dispatch and lifecycle live +// in internal/foundry; the implementations live in internal/casting/<kind>. +// +// This package is a leaf: it is imported by both internal/foundry and every +// internal/casting/<kind> package, so it must not import any of them. The +// only Foundry packages this package may import are api/v1alpha1, +// internal/domain, and internal/tooler. +package planner diff --git a/internal/planner/planner.go b/internal/planner/planner.go new file mode 100644 index 0000000..5791ca9 --- /dev/null +++ b/internal/planner/planner.go @@ -0,0 +1,30 @@ +package planner + +import ( + "context" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/tooler" +) + +// Planner is the per-Kind contract Foundry iterates against. Every Kind +// expresses itself in the same vocabulary: +// +// - identity: Machinery, Patches, Toolers +// - ordering: MoldingKinds (the moldings this Kind processes, in order) +// - stages: EnrichStatus, Mold, MergeStatusIntoSpec +// - lifecycle: Forge, Cast +type Planner interface { + Machinery() v1alpha1.Machinery + Patches() []v1alpha1.PatchEntry + Toolers() []tooler.Tooler + + MoldingKinds() []v1alpha1.MoldingKind + EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind) error + Mold(ctx context.Context, kind v1alpha1.MoldingKind) error + MergeStatusIntoSpec() error + + Forge(ctx context.Context, target string) ([]domain.Material, error) + Cast(ctx context.Context, poursPath string) error +} diff --git a/internal/tooler/clickhousekeepertooler/tooler.go b/internal/tooler/clickhousekeepertooler/tooler.go index 80148e7..986c22f 100644 --- a/internal/tooler/clickhousekeepertooler/tooler.go +++ b/internal/tooler/clickhousekeepertooler/tooler.go @@ -2,7 +2,6 @@ package clickhousekeepertooler import ( "context" - "fmt" "os" root "github.com/hanzoai/o11y-foundry/internal/tooler" @@ -32,7 +31,7 @@ func (tooler *clickhouseKeeperTooler) Gauge(ctx context.Context) error { return nil } - return fmt.Errorf("clickhouse-keeper not found: neither command nor binary at %s", binaryPath) + return errors.Newf(errors.TypeNotFound, "clickhouse-keeper not found: neither command nor binary at %s", binaryPath) } func (tooler *clickhouseKeeperTooler) Install(ctx context.Context) error { diff --git a/internal/tooler/clickhousetooler/tooler.go b/internal/tooler/clickhousetooler/tooler.go index 892c405..0eb1cde 100644 --- a/internal/tooler/clickhousetooler/tooler.go +++ b/internal/tooler/clickhousetooler/tooler.go @@ -2,7 +2,6 @@ package clickhousetooler import ( "context" - "fmt" "os" root "github.com/hanzoai/o11y-foundry/internal/tooler" @@ -32,7 +31,7 @@ func (tooler *clickhouseTooler) Gauge(ctx context.Context) error { return nil } - return fmt.Errorf("clickhouse-server not found: neither command nor binary at %s", binaryPath) + return errors.Newf(errors.TypeNotFound, "clickhouse-server not found: neither command nor binary at %s", binaryPath) } func (tooler *clickhouseTooler) Install(ctx context.Context) error { diff --git a/internal/tooler/dockercomposetooler/tooler.go b/internal/tooler/dockercomposetooler/tooler.go index 8edc02a..d98e524 100644 --- a/internal/tooler/dockercomposetooler/tooler.go +++ b/internal/tooler/dockercomposetooler/tooler.go @@ -2,6 +2,7 @@ package dockercomposetooler import ( "context" + "os/exec" root "github.com/hanzoai/o11y-foundry/internal/tooler" ) @@ -19,7 +20,18 @@ func (tooler *dockerComposeTooler) Name() string { } func (tooler *dockerComposeTooler) Gauge(ctx context.Context) error { - return root.AnyOneExecChecker(ctx, "docker-compose", "docker compose") + // Legacy standalone binary. + if err := root.ExecChecker(ctx, "docker-compose"); err == nil { + return nil + } + + if err := root.ExecChecker(ctx, "docker"); err == nil { + if err := exec.CommandContext(ctx, "docker", "compose", "version").Run(); err == nil { + return nil + } + } + + return errors.Newf(errors.TypeNotFound, "neither 'docker-compose' nor the 'docker compose' plugin is available") } func (tooler *dockerComposeTooler) Install(ctx context.Context) error { diff --git a/internal/tooler/dockerswarmtooler/tooler.go b/internal/tooler/dockerswarmtooler/tooler.go new file mode 100644 index 0000000..12f6af5 --- /dev/null +++ b/internal/tooler/dockerswarmtooler/tooler.go @@ -0,0 +1,57 @@ +package dockerswarmtooler + +import ( + "context" + "errors" + "os/exec" + "strings" + + root "github.com/signoz/foundry/internal/tooler" +) + +var _ root.Tooler = (*dockerSwarmTooler)(nil) + +type dockerSwarmTooler struct{} + +func New() *dockerSwarmTooler { + return &dockerSwarmTooler{} +} + +func (tooler *dockerSwarmTooler) Name() string { + return "docker-swarm" +} + +// Gauge checks that docker is available, swarm mode is active, and the local +// node is a manager (required for docker stack deploy). +func (tooler *dockerSwarmTooler) Gauge(ctx context.Context) error { + if err := root.ExecChecker(ctx, "docker"); err != nil { + return err + } + + // Check swarm is active. + stateCmd := exec.CommandContext(ctx, "docker", "info", "--format", "{{.Swarm.LocalNodeState}}") + stateOut, err := stateCmd.Output() + if err != nil { + return errors.New("failed to check docker swarm status: " + err.Error()) + } + state := strings.TrimSpace(string(stateOut)) + if state != "active" { + return errors.New("docker swarm is not active (state: " + state + "); run 'docker swarm init' to initialize") + } + + // Verify the local node is a manager — stack deploy only works on managers. + roleCmd := exec.CommandContext(ctx, "docker", "info", "--format", "{{.Swarm.ControlAvailable}}") + roleOut, err := roleCmd.Output() + if err != nil { + return errors.New("failed to check docker swarm manager status: " + err.Error()) + } + if strings.TrimSpace(string(roleOut)) != "true" { + return errors.New("current node is a swarm worker, not a manager; 'docker stack deploy' must be run from a manager node") + } + + return nil +} + +func (tooler *dockerSwarmTooler) Install(ctx context.Context) error { + return nil +} diff --git a/internal/tooler/helmtooler/tooler.go b/internal/tooler/helmtooler/tooler.go new file mode 100644 index 0000000..9a9c027 --- /dev/null +++ b/internal/tooler/helmtooler/tooler.go @@ -0,0 +1,27 @@ +package helmtooler + +import ( + "context" + + root "github.com/signoz/foundry/internal/tooler" +) + +var _ root.Tooler = (*helmTooler)(nil) + +type helmTooler struct{} + +func New() *helmTooler { + return &helmTooler{} +} + +func (tooler *helmTooler) Name() string { + return "helm" +} + +func (tooler *helmTooler) Gauge(ctx context.Context) error { + return root.ExecChecker(ctx, "helm") +} + +func (tooler *helmTooler) Install(ctx context.Context) error { + return nil +} diff --git a/internal/tooler/kubectltooler/tooler.go b/internal/tooler/kubectltooler/tooler.go new file mode 100644 index 0000000..7fd5b92 --- /dev/null +++ b/internal/tooler/kubectltooler/tooler.go @@ -0,0 +1,27 @@ +package kubectltooler + +import ( + "context" + + root "github.com/signoz/foundry/internal/tooler" +) + +var _ root.Tooler = (*kubectlTooler)(nil) + +type kubectlTooler struct{} + +func New() *kubectlTooler { + return &kubectlTooler{} +} + +func (t *kubectlTooler) Name() string { + return "kubectl" +} + +func (t *kubectlTooler) Gauge(ctx context.Context) error { + return root.ExecChecker(ctx, "kubectl") +} + +func (t *kubectlTooler) Install(ctx context.Context) error { + return nil +} diff --git a/internal/tooler/postgrestooler/tooler.go b/internal/tooler/postgrestooler/tooler.go index 0573535..ddf0ece 100644 --- a/internal/tooler/postgrestooler/tooler.go +++ b/internal/tooler/postgrestooler/tooler.go @@ -2,7 +2,6 @@ package postgrestooler import ( "context" - "fmt" "os" "path/filepath" @@ -52,7 +51,7 @@ func (tooler *postgresTooler) Gauge(ctx context.Context) error { } } - return fmt.Errorf("postgres not found: neither command nor binary in common locations") + return errors.Newf(errors.TypeNotFound, "postgres not found: neither command nor binary in common locations") } func (tooler *postgresTooler) Install(ctx context.Context) error { diff --git a/internal/tooler/terraformtooler/tooler.go b/internal/tooler/terraformtooler/tooler.go new file mode 100644 index 0000000..765fc65 --- /dev/null +++ b/internal/tooler/terraformtooler/tooler.go @@ -0,0 +1,30 @@ +package terraformtooler + +import ( + "context" + + root "github.com/signoz/foundry/internal/tooler" +) + +var _ root.Tooler = (*terraformTooler)(nil) + +type terraformTooler struct{} + +func New() *terraformTooler { + return &terraformTooler{} +} + +func (t *terraformTooler) Name() string { + return "terraform" +} + +func (t *terraformTooler) Gauge(ctx context.Context) error { + return root.ExecChecker(ctx, "terraform") +} + +func (t *terraformTooler) Install(ctx context.Context) error { + // Terraform installation is platform-specific and typically requires manual installation + // or use of a package manager. We return nil here as users are expected to have + // terraform installed. + return nil +} diff --git a/internal/types/address.go b/internal/types/address.go deleted file mode 100644 index 612ff5e..0000000 --- a/internal/types/address.go +++ /dev/null @@ -1,63 +0,0 @@ -package types - -import ( - "fmt" - "net/url" - "strconv" -) - -type Address struct { - host string - port int -} - -func (address *Address) Host() string { - return address.host -} - -func (address *Address) Port() int { - return address.port -} - -// FormatAddress creates a formatted address string from scheme, host, and port. -func FormatAddress(scheme, host string, port int) string { - return fmt.Sprintf("%s://%s:%d", scheme, host, port) -} - -// NewAddress parses a URL-formatted address string into host and port components. -func NewAddress(address string) (Address, error) { - u, err := url.Parse(address) - if err != nil { - return Address{}, fmt.Errorf("invalid address %q: %w", address, err) - } - - host := u.Hostname() - portStr := u.Port() - - if host == "" { - return Address{}, fmt.Errorf("address %q has no host", address) - } - - var port int - if portStr != "" { - port, err = strconv.Atoi(portStr) - if err != nil { - return Address{}, fmt.Errorf("address %q has invalid port: %w", address, err) - } - } - - return Address{host: host, port: port}, nil -} - -// NewAddresses parses multiple address strings and returns the parsed results. -func NewAddresses(addresses []string) ([]Address, error) { - result := make([]Address, 0, len(addresses)) - for i, addr := range addresses { - parsed, err := NewAddress(addr) - if err != nil { - return nil, fmt.Errorf("address[%d]: %w", i, err) - } - result = append(result, parsed) - } - return result, nil -} diff --git a/internal/types/format.go b/internal/types/format.go deleted file mode 100644 index 6a1caf8..0000000 --- a/internal/types/format.go +++ /dev/null @@ -1,10 +0,0 @@ -package types - -var ( - FormatYAML Format = Format{s: "yaml"} - FormatJSON Format = Format{s: "json"} - FormatINI Format = Format{s: "ini"} - FormatText Format = Format{s: "text"} -) - -type Format struct{ s string } diff --git a/internal/types/ini.go b/internal/types/ini.go deleted file mode 100644 index 01c1bc2..0000000 --- a/internal/types/ini.go +++ /dev/null @@ -1,112 +0,0 @@ -package types - -import ( - "bytes" - "encoding/json" - "fmt" - "gopkg.in/ini.v1" - "slices" -) - -// INI/Env bytes -> JSON bytes. -func INIToJSON(contents []byte) ([]byte, error) { - cfg, err := ini.LoadSources(ini.LoadOptions{ - AllowShadows: true, - }, contents) - if err != nil { - return nil, err - } - - data := make(map[string]map[string]any) - - for _, section := range cfg.Sections() { - // Skip the default section if it's empty (common in systemd files) - if section.Name() == ini.DefaultSection && len(section.Keys()) == 0 { - continue - } - - sectionData := make(map[string]any) - for _, key := range section.Keys() { - // Use ValueWithShadows() to get all values for this key - vals := key.ValueWithShadows() - - if len(vals) > 1 { - // If multiple values exist, store as an array - sectionData[key.Name()] = vals - } else { - // If only one value exists, store as a string - sectionData[key.Name()] = key.String() - } - } - data[section.Name()] = sectionData - } - - return json.MarshalIndent(data, "", " ") -} - -// JSONToINI converts JSON to Systemd INI format. -func JSONToINI(contents []byte) ([]byte, error) { - var data map[string]map[string]any - if err := json.Unmarshal(contents, &data); err != nil { - return nil, err - } - - cfg, err := ini.LoadSources(ini.LoadOptions{ - AllowShadows: true, - PreserveSurroundedQuote: true, - }, []byte("")) - - if err != nil { - return nil, err - } - ini.PrettyFormat = false - // Process Sections - for _, sName := range getSortedKeys(data) { - section, _ := cfg.NewSection(sName) - - // Process Keys (Sorted) - for _, kName := range getSortedKeys(data[sName]) { - if err := writeEntry(section, kName, data[sName][kName]); err != nil { - return nil, err - } - } - } - - var buf bytes.Buffer - _, err = cfg.WriteTo(&buf) - return buf.Bytes(), err -} - -// writeEntry adds the "Shadow" (multiple keys). -func writeEntry(sec *ini.Section, key string, value any) error { - if vals, ok := value.([]any); ok { - for i, v := range vals { - strVal := fmt.Sprint(v) - if i == 0 { - if _, err := sec.NewKey(key, strVal); err != nil { - return err - } - } else { - if err := sec.Key(key).AddShadow(strVal); err != nil { - return err - } - } - } - return nil - } - if _, err := sec.NewKey(key, fmt.Sprint(value)); err != nil { - return err - } - return nil -} - -// getSortedKeys is a generic helper that returns keys in consistent order. -// This ensures generated files have stable output regardless of map iteration order. -func getSortedKeys[V any](m map[string]V) []string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - slices.Sort(keys) - return keys -} diff --git a/internal/types/material.go b/internal/types/material.go deleted file mode 100644 index f1e8e17..0000000 --- a/internal/types/material.go +++ /dev/null @@ -1,115 +0,0 @@ -package types - -import ( - "encoding/json" - "fmt" - "github.com/tidwall/gjson" - "k8s.io/apimachinery/pkg/util/yaml" - kyaml "sigs.k8s.io/yaml" -) - -type Material struct { - contents []byte - path string - format Format -} - -func NewMaterial(contents any, path string, format Format) (Material, error) { - contentsBytes, err := json.Marshal(contents) - if err != nil { - return Material{}, fmt.Errorf("failed to marshal contents: %w", err) - } - - return NewYAMLMaterial(contentsBytes, path) -} - -func NewTextMaterial(contents []byte, path string) Material { - return Material{ - contents: contents, - path: path, - format: FormatText, - } -} - -func NewYAMLMaterial(contents []byte, path string) (Material, error) { - jsonContents, err := yaml.ToJSON(contents) - if err != nil { - return Material{}, fmt.Errorf("invalid yaml: %w", err) - } - - return Material{ - contents: jsonContents, - path: path, - format: FormatYAML, - }, nil -} - -func NewINIMaterial(contents []byte, path string) (Material, error) { - jsonContents, err := INIToJSON(contents) - if err != nil { - return Material{}, fmt.Errorf("invalid ini: %w", err) - } - return Material{ - contents: jsonContents, - path: path, - format: FormatINI, - }, nil -} - -func (m Material) Contents() []byte { - return m.contents -} - -func (m Material) FmtContents() []byte { - switch m.format { - case FormatYAML: - fmtContents, err := kyaml.JSONToYAML(m.contents) - if err != nil { - return nil - } - - return fmtContents - case FormatINI: - fmtContents, err := JSONToINI(m.contents) - if err != nil { - return nil - } - return fmtContents - case FormatText: - return m.contents - default: - return m.contents - } -} - -func (m Material) Path() string { - return m.path -} - -func (m Material) GetBytes(path string) ([]byte, error) { - result := gjson.GetBytes(m.contents, path) - if !result.Exists() { - return nil, fmt.Errorf("path %q does not exist", path) - } - - return []byte(result.String()), nil -} - -func (m Material) GetStringSlice(path string) ([]string, error) { - result := gjson.GetBytes(m.contents, path) - - if !result.Exists() { - return nil, fmt.Errorf("path %q does not exist", path) - } - - var items []string - for _, item := range result.Array() { - items = append(items, item.String()) - } - - return items, nil -} - -func (m Material) ToYaml() ([]byte, error) { - return kyaml.JSONToYAML(m.contents) -} diff --git a/internal/types/pointer.go b/internal/types/pointer.go deleted file mode 100644 index 7c61031..0000000 --- a/internal/types/pointer.go +++ /dev/null @@ -1,5 +0,0 @@ -package types - -func NewIntPtr(i int) *int { - return &i -} diff --git a/internal/types/template.go b/internal/types/template.go deleted file mode 100644 index 05ff64a..0000000 --- a/internal/types/template.go +++ /dev/null @@ -1,62 +0,0 @@ -package types - -import ( - "embed" - "io" - "path/filepath" - "text/template" - - "github.com/Masterminds/sprig/v3" -) - -type Template struct { - name string - format Format - tmpl *template.Template -} - -func NewTemplateFromFS(fs embed.FS, path string, format Format) (*Template, error) { - name := filepath.Base(path) - tmpl, err := template.New(name).Funcs(sprig.FuncMap()).ParseFS(fs, path) - if err != nil { - return nil, err - } - - return &Template{name: name, tmpl: tmpl, format: format}, nil -} - -func MustNewTemplateFromFS(fs embed.FS, path string, format Format) *Template { - tmpl, err := NewTemplateFromFS(fs, path, format) - if err != nil { - panic(err) - } - - return tmpl -} - -func NewTemplate(name string, contents []byte) (*Template, error) { - tmpl, err := template.New(name).Funcs(sprig.FuncMap()).Parse(string(contents)) - if err != nil { - return nil, err - } - - return &Template{name: name, tmpl: tmpl}, nil -} - -func MustNewTemplate(name string, contents []byte) *Template { - tmpl, err := NewTemplate(name, contents) - if err != nil { - panic(err) - } - - return tmpl -} - -func (t *Template) Execute(w io.Writer, data any) error { - newtmpl, err := t.tmpl.Clone() - if err != nil { - return err - } - - return newtmpl.ExecuteTemplate(w, t.name, data) -} diff --git a/internal/types/yaml.go b/internal/types/yaml.go deleted file mode 100644 index 211a451..0000000 --- a/internal/types/yaml.go +++ /dev/null @@ -1,31 +0,0 @@ -package types - -import ( - kyaml "sigs.k8s.io/yaml" -) - -func UnmarshalYAML(data []byte, v any) error { - return kyaml.Unmarshal(data, v) -} - -func MustUnmarshalYAML(data []byte, v any) error { - return kyaml.Unmarshal(data, v) -} - -func MarshalYAML(v any) ([]byte, error) { - yaml, err := kyaml.Marshal(v) - if err != nil { - return nil, err - } - - return yaml, nil -} - -func MustMarshalYAML(v any) []byte { - yaml, err := MarshalYAML(v) - if err != nil { - panic(err) - } - - return yaml -} diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 0000000..5821b47 --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,74 @@ +package version + +import ( + "fmt" + "os" + "runtime" + "time" +) + +// These will be set via ldflags at build time. +var ( + version string = "<unset>" + hash string = "<unset>" +) + +var ( + Info Build = Build{ + version: version, + hash: hash, + goVersion: runtime.Version(), + } +) + +// Build contains information about the build environment. +type Build struct { + // The version of the current build. + version string + + // The git hash of the current build. + hash string + + // The version of go. + goVersion string +} + +func (b Build) Version() string { + return b.version +} + +func (b Build) Hash() string { + return b.hash +} + +func (b Build) GoVersion() string { + return b.goVersion +} + +func (b Build) Short() string { + return fmt.Sprintf("foundryctl %s (%s) %s", b.version, b.hash, b.goVersion) +} + +func (b Build) PrettyPrint() { + year := time.Now().Year() + ascii := []string{ + "███████╗ ██████╗ ██╗ ██╗███╗ ██╗██████╗ ██████╗ ██╗ ██╗", + "██╔════╝██╔═══██╗██║ ██║████╗ ██║██╔══██╗██╔══██╗╚██╗ ██╔╝", + "█████╗ ██║ ██║██║ ██║██╔██╗ ██║██║ ██║██████╔╝ ╚████╔╝ ", + "██╔══╝ ██║ ██║██║ ██║██║╚██╗██║██║ ██║██╔══██╗ ╚██╔╝ ", + "██║ ╚██████╔╝╚██████╔╝██║ ╚████║██████╔╝██║ ██║ ██║ ", + "╚═╝ ╚═════╝ ╚═════╝ ╚═╝ ╚═══╝╚═════╝ ╚═╝ ╚═╝ ╚═╝ ", + } + + _, _ = fmt.Fprintln(os.Stdout) + for _, line := range ascii { + _, _ = fmt.Fprintln(os.Stdout, line) + } + _, _ = fmt.Fprintln(os.Stdout) + _, _ = fmt.Fprintf(os.Stdout, " Version: %s\n", b.version) + _, _ = fmt.Fprintf(os.Stdout, " Commit: %s\n", b.hash) + _, _ = fmt.Fprintf(os.Stdout, " Go: %s\n", b.goVersion) + _, _ = fmt.Fprintln(os.Stdout) + _, _ = fmt.Fprintf(os.Stdout, " Copyright %d SigNoz, All rights reserved.\n", year) + _, _ = fmt.Fprintln(os.Stdout) +} diff --git a/internal/writer/output.go b/internal/writer/output.go new file mode 100644 index 0000000..c077781 --- /dev/null +++ b/internal/writer/output.go @@ -0,0 +1,21 @@ +package writer + +import ( + "encoding/json" + "io" +) + +// WriteOutput writes m's MarshalJSON bytes (with a trailing newline) to w. +// Used for stream payloads that don't have a filesystem path. Each implementer +// owns its own envelope (e.g. errors.Envelope wraps as {"exception": {...}}) +// via MarshalJSON, so the writer stays a thin transport. +func WriteOutput(w io.Writer, m json.Marshaler) error { + data, err := m.MarshalJSON() + if err != nil { + return err + } + + _, err = w.Write(append(data, '\n')) + + return err +} diff --git a/internal/writer/writer.go b/internal/writer/writer.go index f295a02..f19252d 100644 --- a/internal/writer/writer.go +++ b/internal/writer/writer.go @@ -2,7 +2,6 @@ package writer import ( "context" - "fmt" "io" "log/slog" "os" @@ -37,7 +36,7 @@ func New(logger *slog.Logger, options *Options) (*Writer, error) { } if err := os.MkdirAll(options.TargetDirectory, 0755); err != nil { - return nil, fmt.Errorf("failed to create output directory '%s': %s", options.TargetDirectory, err.Error()) + return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to create output directory '%s'", options.TargetDirectory) } return &Writer{ @@ -46,7 +45,7 @@ func New(logger *slog.Logger, options *Options) (*Writer, error) { }, nil } -func (w *Writer) Write(ctx context.Context, material types.Material) error { +func (w *Writer) Write(ctx context.Context, material domain.Material) error { if _, ok := w.options.Output.(*os.File); ok { path := filepath.Join(w.options.TargetDirectory, material.Path()) @@ -75,7 +74,7 @@ func (w *Writer) Write(ctx context.Context, material types.Material) error { return nil } -func (w *Writer) WriteMany(ctx context.Context, materials ...types.Material) error { +func (w *Writer) WriteMany(ctx context.Context, materials ...domain.Material) error { for _, material := range materials { if err := w.Write(ctx, material); err != nil { return err diff --git a/scripts/foundry.sh b/scripts/foundry.sh new file mode 100755 index 0000000..c6a461b --- /dev/null +++ b/scripts/foundry.sh @@ -0,0 +1,361 @@ +#!/bin/bash +# foundry.sh - Downloads, verifies, and installs the foundryctl binary +# from GitHub releases. +# +# Usage: +# curl -fsSL https://signoz.io/foundry.sh | bash +# curl -fsSL https://signoz.io/foundry.sh | FOUNDRY_VERSION=v0.1.4 bash +# bash foundry.sh -v v0.1.4 +# bash foundry.sh -d /usr/local/bin +# bash foundry.sh -h +# +# OPTIONS: +# -v <version> Version to install (e.g. v0.1.4). Default: latest. +# -d <path> Install directory. Default: $XDG_BIN_HOME or ~/.local/bin. +# -y Auto-confirm upgrade prompt. +# -h Show help message. +# +# Environment: +# FOUNDRY_VERSION Equivalent to -v. +# FOUNDRY_INSTALL_DIR Equivalent to -d. +# FOUNDRY_ASSUME_YES Equivalent to -y. Set to "true" to enable. +# NO_COLOR When set, disables ANSI color output (https://no-color.org). + +set -euo pipefail + +# Constants. +readonly NAME="foundry.sh" +readonly REPO="SigNoz/foundry" +readonly BINARY="foundryctl" + +# User input (env vars; flags overwrite these in the getopts loop below). +FOUNDRY_VERSION="${FOUNDRY_VERSION:-}" +FOUNDRY_INSTALL_DIR="${FOUNDRY_INSTALL_DIR:-}" +FOUNDRY_ASSUME_YES="${FOUNDRY_ASSUME_YES:-false}" + +# https://no-color.org honoured; auto-stripped when stderr is not a TTY. +if [[ -t 2 ]] && [[ -z "${NO_COLOR:-}" ]]; then + readonly C_INFO=$'\033[32;1m' + readonly C_WARN=$'\033[33;1m' + readonly C_ERROR=$'\033[31;1m' + readonly C_RESET=$'\033[0m' +else + readonly C_INFO="" + readonly C_WARN="" + readonly C_ERROR="" + readonly C_RESET="" +fi + +info() { + echo "${C_INFO}[INFO]${C_RESET} $*" +} + +warn() { + echo "${C_WARN}[WARN]${C_RESET} $*" >&2 +} + +err() { + echo "${C_ERROR}[ERROR]${C_RESET} $*" >&2 +} + +die() { + err "$*" + exit 1 +} + +help() { + printf "NAME\n" + printf "\t%s - Install %s, the SigNoz Foundry CLI\n\n" "${NAME}" "${BINARY}" + printf "USAGE\n" + printf "\t%s [-v version] [-d directory] [-y] [-h]\n\n" "${NAME}" + printf "DESCRIPTION\n" + printf "\tDownloads, verifies, and installs the %s binary from GitHub releases.\n\n" "${BINARY}" + printf "OPTIONS\n" + printf "\t-v <version>\tVersion to install (e.g. v0.1.4). [env: FOUNDRY_VERSION] [default: latest]\n" + printf "\t-d <path>\tInstall directory. [env: FOUNDRY_INSTALL_DIR] [default: \$XDG_BIN_HOME or ~/.local/bin]\n" + printf "\t-y\t\tAuto-confirm upgrade prompt. [env: FOUNDRY_ASSUME_YES]\n" + printf "\t-h\t\tShow this help message.\n\n" + printf "EXAMPLES\n" + printf "\t%s -v v0.1.4\n" "${NAME}" + printf "\t%s -d /usr/local/bin\n" "${NAME}" + printf "\tcurl -fsSL https://signoz.io/foundry.sh | bash\n" + printf "\tcurl -fsSL https://signoz.io/foundry.sh | FOUNDRY_VERSION=v0.1.4 bash\n" +} + +# Sets PLATFORM_* (OS, ARCH, BIN_SUFFIX); Windows shells map to OS=windows and .exe suffix. +init_platform() { + local raw_arch raw_os + raw_arch="$(uname -m)" + case "${raw_arch}" in + x86_64 | amd64) PLATFORM_ARCH="amd64" ;; + aarch64 | arm64) PLATFORM_ARCH="arm64" ;; + *) die "Unsupported architecture: ${raw_arch}" ;; + esac + + raw_os="$(uname | tr '[:upper:]' '[:lower:]')" + PLATFORM_BIN_SUFFIX="" + case "${raw_os}" in + darwin | linux) PLATFORM_OS="${raw_os}" ;; + mingw* | cygwin* | msys*) + PLATFORM_OS="windows" + PLATFORM_BIN_SUFFIX=".exe" + ;; + *) die "Unsupported operating system: ${raw_os}" ;; + esac +} + +# Sets HAS_CURL/HAS_WGET and SHA256_CMD for downstream reuse. +verify_prereqs() { + HAS_CURL="$(command -v curl >/dev/null 2>&1 && echo true || echo false)" + HAS_WGET="$(command -v wget >/dev/null 2>&1 && echo true || echo false)" + if [[ "${HAS_CURL}" != "true" ]] && [[ "${HAS_WGET}" != "true" ]]; then + die "Missing prerequisite: curl or wget" + fi + local cmd + for cmd in tar mktemp install; do + if ! command -v "${cmd}" >/dev/null 2>&1; then + die "Missing prerequisite: ${cmd}" + fi + done + if command -v sha256sum >/dev/null 2>&1; then + SHA256_CMD="sha256sum" + elif command -v shasum >/dev/null 2>&1; then + SHA256_CMD="shasum -a 256" + else + die "Missing prerequisite: sha256sum or shasum" + fi +} + +# fetch downloads URL to OUT using whichever of curl/wget is available. +# Pass "progress" as the third arg to render a transfer progress bar; the bar +# is suppressed when stderr is not a TTY (e.g. CI, piped output). +fetch() { + local url="$1" + local out="$2" + local mode="${3:-quiet}" + if [[ "${HAS_CURL}" == "true" ]]; then + if [[ "${mode}" == "progress" ]] && [[ -t 2 ]]; then + curl -fL --progress-bar "${url}" -o "${out}" + else + curl -fsSL "${url}" -o "${out}" + fi + else + if [[ "${mode}" == "progress" ]] && [[ -t 2 ]]; then + wget -q --show-progress -O "${out}" "${url}" + else + wget -q -O "${out}" "${url}" + fi + fi +} + +# fetch_effective_url follows redirects on URL and prints the final location. +fetch_effective_url() { + local url="$1" + if [[ "${HAS_CURL}" == "true" ]]; then + curl -sIL -o /dev/null -w '%{url_effective}' "${url}" + else + wget --max-redirect=5 --server-response --spider "${url}" 2>&1 \ + | awk '/^ Location: /{u=$2} END{print u}' + fi +} + +# Sets TAG from FOUNDRY_VERSION or /releases/latest redirect; validates semver shape. +resolve_version() { + if [[ -n "${FOUNDRY_VERSION}" ]]; then + case "${FOUNDRY_VERSION}" in + v*) TAG="${FOUNDRY_VERSION}" ;; + *) TAG="v${FOUNDRY_VERSION}" ;; + esac + else + local latest_url="https://github.com/${REPO}/releases/latest" + local resolved + resolved="$(fetch_effective_url "${latest_url}")" + TAG="${resolved##*/tag/}" + if [[ -z "${TAG}" ]] || [[ "${TAG}" == "${resolved}" ]]; then + die "Could not resolve latest release tag from ${latest_url}" + fi + fi + + if [[ ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then + die "Invalid version: ${TAG} (expected vMAJOR.MINOR.PATCH)" + fi +} + +resolve_install_dir() { + if [[ -n "${FOUNDRY_INSTALL_DIR}" ]]; then + INSTALL_DIR="${FOUNDRY_INSTALL_DIR}" + elif [[ -n "${XDG_BIN_HOME:-}" ]]; then + INSTALL_DIR="${XDG_BIN_HOME}" + else + INSTALL_DIR="${HOME}/.local/bin" + fi + mkdir -p "${INSTALL_DIR}" + if [[ ! -w "${INSTALL_DIR}" ]]; then + err "Install directory is not writable: ${INSTALL_DIR}" + die "Set FOUNDRY_INSTALL_DIR to a writable path or run with appropriate permissions." + fi + DEST="${INSTALL_DIR}/${BINARY}${PLATFORM_BIN_SUFFIX}" +} + +# Same version: skip and exit 0. Different version: prompt if interactive, +# auto-proceed if piped (curl-pipe-bash, CI). +check_existing() { + if [[ ! -f "${DEST}" ]]; then + return + fi + local current + current="$("${DEST}" version 2>/dev/null | awk '/^[[:space:]]*Version:/ {print $NF; exit}' || true)" + if [[ -z "${current}" ]]; then + return + fi + if [[ "${current}" == "${TAG}" ]]; then + info "${BINARY} ${TAG} is already installed at ${DEST}" + exit 0 + fi + if [[ "${FOUNDRY_ASSUME_YES}" == "true" ]] || [[ ! -t 0 ]]; then + info "Updating ${BINARY} ${current} -> ${TAG}" + return + fi + local answer + read -r -p "Update ${BINARY} ${current} to ${TAG}? [Y/n] " answer + case "${answer}" in + "" | y | Y | yes | YES) ;; + *) die "Aborted by user" ;; + esac +} + +# Sets TARBALL, CHECKSUMS, TARBALL_URL, CHECKSUMS_URL from PLATFORM_* and TAG. +compute_release_artifacts() { + TARBALL="foundry_${PLATFORM_OS}_${PLATFORM_ARCH}.tar.gz" + CHECKSUMS="foundry_${TAG#v}_checksums.txt" + TARBALL_URL="https://github.com/${REPO}/releases/download/${TAG}/${TARBALL}" + CHECKSUMS_URL="https://github.com/${REPO}/releases/download/${TAG}/${CHECKSUMS}" +} + +# Creates TMP_ROOT and fetches the tarball and checksums into it. +download_release() { + TMP_ROOT="$(mktemp -d -t foundry-installer-XXXXXX)" + TARBALL_PATH="${TMP_ROOT}/${TARBALL}" + CHECKSUMS_PATH="${TMP_ROOT}/${CHECKSUMS}" + + info "Downloading ${TARBALL_URL}" + fetch "${TARBALL_URL}" "${TARBALL_PATH}" progress + fetch "${CHECKSUMS_URL}" "${CHECKSUMS_PATH}" +} + +# Checksums file lists "<sha256> <filename>" or "<sha256> *<filename>". +verify_checksum() { + local expected + expected="$(awk -v f="${TARBALL}" '$2 == f || $2 == "*"f {print $1; exit}' "${CHECKSUMS_PATH}")" + if [[ -z "${expected}" ]]; then + die "Checksum for ${TARBALL} not found in ${CHECKSUMS}" + fi + + local actual + actual="$(${SHA256_CMD} "${TARBALL_PATH}" | awk '{print $1}')" + + if [[ "${expected}" != "${actual}" ]]; then + err "Checksum mismatch for ${TARBALL}" + err " expected: ${expected}" + die " actual: ${actual}" + fi +} + +install_binary() { + local extract_dir="${TMP_ROOT}/extract" + mkdir -p "${extract_dir}" + tar -xzf "${TARBALL_PATH}" -C "${extract_dir}" + + local src="${extract_dir}/foundry_${PLATFORM_OS}_${PLATFORM_ARCH}/bin/${BINARY}${PLATFORM_BIN_SUFFIX}" + if [[ ! -f "${src}" ]]; then + die "Expected binary not found in tarball: ${src#"${extract_dir}"/}" + fi + + install -m 0755 "${src}" "${DEST}" + info "Installed ${BINARY}${PLATFORM_BIN_SUFFIX} ${TAG} to ${DEST}" +} + +# Smoke test: catches arch/platform mismatches that slipped past checksum. +verify_install() { + local output + if ! output="$("${DEST}" version 2>&1)"; then + err "Installed binary failed to run: ${DEST}" + err "This may indicate a wrong-arch download or a permissions issue." + die "${output}" + fi + echo + echo "${output}" +} + +print_path_hint() { + case ":${PATH}:" in + *":${INSTALL_DIR}:"*) return ;; + *) ;; + esac + + local rc_file + # shellcheck disable=SC2088 + case "${SHELL:-}" in + */zsh) rc_file='~/.zshrc' ;; + */bash) rc_file='~/.bashrc (Linux) or ~/.bash_profile (macOS)' ;; + */fish) rc_file='~/.config/fish/config.fish' ;; + *) rc_file='your shell config' ;; + esac + + echo + warn "${INSTALL_DIR} is not on your PATH." + echo "To use ${BINARY} from any shell, add this to ${rc_file}:" + echo + echo " export PATH=\"${INSTALL_DIR}:\$PATH\"" +} + +cleanup() { + if [[ -n "${TMP_ROOT:-}" ]] && [[ -d "${TMP_ROOT}" ]]; then + rm -rf "${TMP_ROOT}" + fi +} + +fail_trap() { + local rc=$? + if [[ ${rc} -ne 0 ]]; then + err "Failed to install ${BINARY}." + err "For support, see https://github.com/${REPO}/issues" + fi + cleanup + exit "${rc}" +} + +run() { + init_platform + verify_prereqs + resolve_version + resolve_install_dir + check_existing + compute_release_artifacts + download_release + verify_checksum + install_binary + verify_install + print_path_hint + cleanup +} + +trap fail_trap EXIT + +while getopts 'v:d:yh' opt; do + case "${opt}" in + v) FOUNDRY_VERSION="${OPTARG:-}" ;; + d) FOUNDRY_INSTALL_DIR="${OPTARG:-}" ;; + y) FOUNDRY_ASSUME_YES="true" ;; + h) + help + trap - EXIT + exit 0 + ;; + ?) die "Invalid option: -${OPTARG:-}" ;; + *) die "Unknown error while processing options" ;; + esac +done + +run +trap - EXIT