diff --git a/.github/workflows/chart-pr.yml b/.github/workflows/chart-pr.yml new file mode 100644 index 0000000000..114203bb19 --- /dev/null +++ b/.github/workflows/chart-pr.yml @@ -0,0 +1,252 @@ +name: Chart PR release + +on: + workflow_dispatch: + inputs: + pr_number: + description: 'PR number (used for chart versioning and PR comment)' + required: true + type: string + +env: + REGISTRY: us-docker.pkg.dev + IMAGE_PREFIX: castai-hub/library/liqo + GHCR_REGISTRY: ghcr.io + GHCR_IMAGE_PREFIX: castai/images/liqo + +jobs: + build-go: + name: Build Go image + runs-on: ubuntu-latest + environment: pr-build + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + component: + - crd-replicator + - ipam + - liqo-controller-manager + - webhook + - uninstaller + - virtual-kubelet + - metric-agent + - telemetry + - proxy + - gateway + - gateway/wireguard + - gateway/geneve + - fabric + env: + DOCKER_REGISTRY: us-docker.pkg.dev + DOCKER_ORGANIZATION: castai-hub/library/liqo + DOCKER_TAG: ${{ github.sha }} + DOCKER_PUSH: "true" + ARCHS: linux/amd64,linux/arm64 + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: go.sum + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: _json_key + password: ${{ secrets.GCP_IMAGE_RELEASER_JSON_KEY }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.GHCR_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push ${{ matrix.component }} image + run: | + chmod +x ./build/liqo/build.sh + ./build/liqo/build.sh ./cmd/${{ matrix.component }} + + build-standard: + name: Build standard image + runs-on: ubuntu-latest + environment: pr-build + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + component: + - cert-creator + - liqo-crd-upgrade + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: _json_key + password: ${{ secrets.GCP_IMAGE_RELEASER_JSON_KEY }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.GHCR_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Cache Docker layers + uses: actions/cache@v4 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-${{ matrix.component }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ matrix.component }}-buildx- + + - name: Configure the build-push-action dockerfile path + id: dockerfile + run: | + if [[ -d "build/${{ matrix.component }}" ]]; then + echo "path=build/${{ matrix.component }}/Dockerfile" >> "$GITHUB_OUTPUT" + else + echo "path=build/common/Dockerfile" >> "$GITHUB_OUTPUT" + fi + + - name: Build (and Publish) ${{ matrix.component }} image + uses: docker/build-push-action@v6 + env: + DOCKER_BUILD_RECORD_UPLOAD: false + with: + context: . + platforms: linux/amd64 + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/${{ matrix.component }}-ci:latest + ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/${{ matrix.component }}-ci:${{ github.sha }} + ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE_PREFIX }}/${{ matrix.component }}-ci:latest + ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE_PREFIX }}/${{ matrix.component }}-ci:${{ github.sha }} + push: true + file: ${{ steps.dockerfile.outputs.path }} + build-args: | + COMPONENT=${{ matrix.component }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + publish-chart: + name: Publish PR chart + runs-on: ubuntu-latest + environment: pr-build + needs: [build-go, build-standard] + permissions: + contents: read + packages: write + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # fetch-depth 0 is required for `git describe --tags` (appVersion). + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v4.2.0 + with: + version: v3.17.0 + + - name: Login to GitHub Container Registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Package and push chart + run: | + PR_NUMBER="${{ inputs.pr_number }}" + # The commit SHA this workflow was dispatched on; used as the image tag. + # Because the liqo chart's `liqo.suffix` helper appends `-ci` to image + # repos when the tag is not a semver string, this resolves each image to + # the `-ci:` build produced by the build jobs above. + IMAGE_TAG="${{ github.sha }}" + APP_VERSION="$(git describe --tags)" + + # Bump the chart version with the PR number (overwritten on each run). + sed -i "s/^version:.*/version: 0.0.0-pr${PR_NUMBER}/" deployments/liqo/Chart.yaml + sed -i "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/" deployments/liqo/Chart.yaml + + # Point all Liqo images to this PR's CI build. + sed -i "s/^tag:.*/tag: \"${IMAGE_TAG}\"/" deployments/liqo/values.yaml + + # Update the liqo-crds subchart's appVersion so the CRD upgrade job + # image resolves to the correct CI build. + sed -i "s/^appVersion:.*/appVersion: \"${IMAGE_TAG}\"/" deployments/liqo/charts/liqo-crds/Chart.yaml + + # Build the file:// subchart dependency (liqo-crds) before packaging. + helm dependency build deployments/liqo + + helm package deployments/liqo + helm push liqo-0.0.0-pr${PR_NUMBER}.tgz oci://ghcr.io/${{ github.repository }}/charts + + - name: Comment PR with install instructions + uses: actions/github-script@v7 + with: + script: | + const prNumber = parseInt("${{ inputs.pr_number }}", 10); + const repo = `${context.repo.owner}/${context.repo.repo}`.toLowerCase(); + const commitSha = "${{ github.sha }}"; + + // Reuse an existing comment if we already posted one, to avoid spam. + const comments = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber + }); + + const botComment = comments.data.find(c => c.body.includes('## Helm Chart Published')); + const body = `## Helm Chart Published + + Install this PR's chart for testing: + \`\`\`bash + helm install liqo oci://ghcr.io/${repo}/charts/liqo --version 0.0.0-pr${prNumber} + \`\`\` + + This chart references the Liqo CI images built for commit \`${commitSha}\`.`; + + if (botComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: body + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: body + }); + } diff --git a/.github/workflows/greeting.yml b/.github/workflows/greeting.yml deleted file mode 100644 index 32dbd760d0..0000000000 --- a/.github/workflows/greeting.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Greeting -on: - pull_request_target: - types: - - opened - -jobs: - greeting: - name: Issue a greeting comment upon PR - runs-on: ubuntu-latest - - steps: - - name: Issue the greeting comment - uses: peter-evans/create-or-update-comment@v4 - with: - token: ${{ secrets.CI_TOKEN }} - issue-number: ${{ github.event.pull_request.number }} - body: | - Hi @${{ github.event.pull_request.user.login }}. Thanks for your PR! - - I am @adamjensenbot. - You can interact with me issuing a **slash command** in the first line of a **comment**. - Currently, I understand the following commands: - * `/rebase`: Rebase this PR onto the master branch (You can add the option `test=true` to launch the tests - when the rebase operation is completed) - * `/merge`: Merge this PR into the master branch - * `/build` Build Liqo components - * `/test` Launch the E2E and Unit tests - * `/hold`, `/unhold` Add/remove the hold label to prevent merging with /merge - - Make sure this PR appears in the **${{ github.event.repository.name }} changelog**, adding one of the following **labels**: - * `feat`: :rocket: New Feature - * `fix`: :bug: Bug Fix - * `refactor`: :broom: Code Refactoring - * `docs`: :memo: Documentation - * `style`: :lipstick: Code Style - * `perf`: :racehorse: Performance Improvement - * `test`: :white_check_mark: Tests - * `chore`: :truck: Dependencies Management - * `build`: :package: Builds Management - * `ci`: :construction_worker: CI/CD - * `revert`: :rewind: Reverts Previous Changes - diff --git a/.github/workflows/hold.yml b/.github/workflows/hold.yml deleted file mode 100644 index 9814d01047..0000000000 --- a/.github/workflows/hold.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Manage the hold label -on: - repository_dispatch: - types: - - hold-command - - unhold-command - -jobs: - hold: - name: Add hold label - runs-on: ubuntu-latest - if: github.event.action == 'hold-command' - - steps: - - name: Add the hold label to avoid merging - uses: actions-ecosystem/action-add-labels@v1 - with: - github_token: "${{ secrets.CI_TOKEN }}" - number: ${{ github.event.client_payload.github.payload.issue.number }} - labels: hold - - - name: Report status as reaction - uses: peter-evans/create-or-update-comment@v4 - with: - token: ${{ secrets.CI_TOKEN }} - comment-id: ${{ github.event.client_payload.github.payload.comment.id }} - reactions: hooray - - unhold: - name: Remove the hold label - runs-on: ubuntu-latest - if: github.event.action == 'unhold-command' - - steps: - - name: Remove the hold label to allow merging - uses: actions-ecosystem/action-remove-labels@v1 - with: - github_token: "${{ secrets.CI_TOKEN }}" - number: ${{ github.event.client_payload.github.payload.issue.number }} - labels: hold - - - name: Report status as reaction - uses: peter-evans/create-or-update-comment@v4 - with: - token: ${{ secrets.CI_TOKEN }} - comment-id: ${{ github.event.client_payload.github.payload.comment.id }} - reactions: hooray diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 3b6862871e..e2456afdcb 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -5,14 +5,21 @@ on: - "v*" branches: - master - repository_dispatch: + issue_comment: types: - - test-command - - build-command + - created + +env: + REGISTRY: us-docker.pkg.dev + IMAGE_PREFIX: castai-hub/library/liqo + GHCR_REGISTRY: ghcr.io + GHCR_IMAGE_PREFIX: castai/images/liqo + jobs: configure: name: Preliminary configuration runs-on: ubuntu-latest + if: github.event_name != 'issue_comment' || (github.event_name == 'issue_comment' && contains(github.event.comment.body, '/build')) outputs: commit-ref: ${{ steps.configure.outputs.commit-ref }} repo-suffix: ${{ steps.configure.outputs.repo-suffix }} @@ -20,6 +27,8 @@ jobs: repo-name: ${{ steps.configure.outputs.repo-name }} architectures: ${{ steps.configure.outputs.architectures }} pr-number: ${{ steps.configure.outputs.pr-number }} + registry: ${{ steps.configure.outputs.registry }} + image-prefix: ${{ steps.configure.outputs.image-prefix }} steps: - name: Get the version id: get_version @@ -32,14 +41,14 @@ jobs: if [ "${{ github.event_name }}" == "repository_dispatch" ]; then { echo "master=false" - echo "architectures=linux/amd64,linux/arm64" + echo "architectures=linux/amd64" echo "commit-ref=${{ github.event.client_payload.pull_request.head.sha }}" echo "pr-number=${{ github.event.client_payload.github.payload.issue.number }}" } >> "$GITHUB_OUTPUT" elif [ "${{ steps.get_version.outputs.VERSION }}" != "" ]; then { echo "master=false" - echo "architectures=linux/amd64,linux/arm64,linux/arm/v7" + echo "architectures=linux/amd64,linux/arm64" echo "commit-ref=${{ steps.get_version.outputs.VERSION }}" } >> "$GITHUB_OUTPUT" elif [ "${{ github.ref_name }}" == "master" ]; then @@ -67,6 +76,10 @@ jobs: echo "repo-name=${{ github.event.client_payload.github.payload.repository.full_name }}" >> "$GITHUB_OUTPUT" fi + # Set registry configuration + echo "registry=${{ env.REGISTRY }}" >> "$GITHUB_OUTPUT" + echo "image-prefix=${{ env.IMAGE_PREFIX }}" >> "$GITHUB_OUTPUT" + # Since we are using a repository-dispatch event, we have to explicitly set a run check. We initialize it to a "pending" state. - uses: octokit/request-action@v2.x name: "Initialize run check to 'pending (For PR-only)" @@ -83,6 +96,9 @@ jobs: build-standard: runs-on: ubuntu-latest needs: configure + permissions: + contents: read + packages: write strategy: fail-fast: false matrix: @@ -114,6 +130,20 @@ jobs: repository: "${{ needs.configure.outputs.repo-name }}" persist-credentials: false + - name: Login to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: _json_key + password: ${{ secrets.GCP_IMAGE_RELEASER_JSON_KEY }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.GHCR_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Cache Docker layers uses: actions/cache@v4 with: @@ -122,13 +152,6 @@ jobs: restore-keys: | ${{ runner.os }}-${{ matrix.component }}-buildx- - - name: Login to the Container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Configure the build-push-action dockerfile path id: dockerfile run: | @@ -144,8 +167,10 @@ jobs: context: . platforms: ${{ env.ARCHITECTURES }} tags: | - ghcr.io/${{ github.repository_owner }}/${{ matrix.component }}${{ needs.configure.outputs.repo-suffix }}:latest - ghcr.io/${{ github.repository_owner }}/${{ matrix.component }}${{ needs.configure.outputs.repo-suffix }}:${{ needs.configure.outputs.commit-ref }} + ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/${{ matrix.component }}${{ needs.configure.outputs.repo-suffix }}:latest + ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/${{ matrix.component }}${{ needs.configure.outputs.repo-suffix }}:${{ needs.configure.outputs.commit-ref }} + ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE_PREFIX }}/${{ matrix.component }}${{ needs.configure.outputs.repo-suffix }}:latest + ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE_PREFIX }}/${{ matrix.component }}${{ needs.configure.outputs.repo-suffix }}:${{ needs.configure.outputs.commit-ref }} push: true file: ${{ steps.dockerfile.outputs.path }} build-args: | @@ -165,9 +190,12 @@ jobs: build-go: runs-on: ubuntu-latest needs: configure + permissions: + contents: read + packages: write env: - DOCKER_REGISTRY: ghcr.io - DOCKER_ORGANIZATION: ${{ github.repository_owner }} + DOCKER_REGISTRY: ${{ needs.configure.outputs.registry }} + DOCKER_ORGANIZATION: ${{ needs.configure.outputs.image-prefix }} DOCKER_TAG: ${{ needs.configure.outputs.commit-ref }} DOCKER_PUSH: true ARCHS: ${{ needs.configure.outputs.architectures }} @@ -209,7 +237,14 @@ jobs: - name: Login to the Container registry uses: docker/login-action@v3 with: - registry: ghcr.io + registry: ${{ env.REGISTRY }} + username: _json_key + password: ${{ secrets.GCP_IMAGE_RELEASER_JSON_KEY }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.GHCR_REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} @@ -340,7 +375,7 @@ jobs: id: last-release with: repository: ${{ github.repository }} - excludes: prerelease, draft + excludes: draft - name: Configure Git run: | @@ -390,16 +425,15 @@ jobs: with: path: ./liqoctl/ - # we need to upload only helm artifacts first, to make the index updater to work + # we need to upload only helm artifacts first, to make the index updater to work - uses: ncipollo/release-action@v1 with: artifacts: ".cr-release-packages/*" generateReleaseNotes: true - token: ${{ secrets.CI_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} allowUpdates: true tag: ${{ needs.configure.outputs.commit-ref }} name: ${{ needs.configure.outputs.commit-ref }} - prerelease: ${{ steps.semver_parser.outputs.prerelease != '' }} - name: Update Helm index run: | @@ -419,11 +453,7 @@ jobs: - uses: ncipollo/release-action@v1 with: artifacts: "./liqoctl/**/*" - token: ${{ secrets.CI_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} allowUpdates: true tag: ${{ needs.configure.outputs.commit-ref }} name: ${{ needs.configure.outputs.commit-ref }} - prerelease: ${{ steps.semver_parser.outputs.prerelease != '' }} - - - name: Update new version in krew-index - uses: rajatjindal/krew-release-bot@v0.0.47 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 90f8137dfa..4da20df1d4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -32,96 +32,92 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: "${{ github.event.pull_request.head.sha }}" - repository: ${{github.event.pull_request.head.repo.full_name}} - persist-credentials: false + - name: Checkout + uses: actions/checkout@v4 + with: + ref: "${{ github.event.pull_request.head.sha }}" + repository: ${{github.event.pull_request.head.repo.full_name}} + persist-credentials: false - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod - - name: Execute go mod tidy and check the outcome - working-directory: ./ - run: | - go mod tidy - git diff --exit-code + - name: Execute go mod tidy and check the outcome + working-directory: ./ + run: | + go mod tidy + git diff --exit-code - - name: Issue a comment in case the of failure - uses: peter-evans/create-or-update-comment@v4 - with: - token: ${{ secrets.CI_TOKEN }} - issue-number: ${{ github.event.pull_request.number }} - body: | - The `go.mod` and/or `go.sum` files appear not to be correctly tidied. + - name: Issue a comment in case the of failure + uses: peter-evans/create-or-update-comment@v4 + with: + token: ${{ secrets.CI_TOKEN }} + issue-number: ${{ github.event.pull_request.number }} + body: | + The `go.mod` and/or `go.sum` files appear not to be correctly tidied. - Please, rerun `go mod tidy` to fix the issues. - reactions: confused - if: | - failure() && github.event.pull_request.head.repo.full_name == github.repository + Please, rerun `go mod tidy` to fix the issues. + reactions: confused + if: | + failure() && github.event.pull_request.head.repo.full_name == github.repository shelllint: name: Lint bash files runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: "${{ github.event.pull_request.head.sha }}" - repository: ${{github.event.pull_request.head.repo.full_name}} - persist-credentials: false - - name: Run Shellcheck - uses: ludeeus/action-shellcheck@2.0.0 - env: - # Allow 'source' outside of FILES - SHELLCHECK_OPTS: -x + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: "${{ github.event.pull_request.head.sha }}" + repository: ${{github.event.pull_request.head.repo.full_name}} + persist-credentials: false + - name: Run Shellcheck + uses: ludeeus/action-shellcheck@2.0.0 + env: + # Allow 'source' outside of FILES + SHELLCHECK_OPTS: -x markdownlint: name: Lint markdown files runs-on: ubuntu-latest steps: - - name: Check out code - uses: actions/checkout@v4 - with: - ref: "${{ github.event.pull_request.head.sha }}" - repository: ${{github.event.pull_request.head.repo.full_name}} - persist-credentials: false - - name: Lint markdown files - uses: avto-dev/markdown-lint@v1 - with: - config: '.markdownlint.yml' - args: '**/*.md' - ignore: ".github/ISSUE_TEMPLATE/*.md docs/usage/liqoctl/*.md" + - name: Check out code + uses: actions/checkout@v4 + with: + ref: "${{ github.event.pull_request.head.sha }}" + repository: ${{github.event.pull_request.head.repo.full_name}} + persist-credentials: false + - name: Lint markdown files + uses: avto-dev/markdown-lint@v1 + with: + config: ".markdownlint.yml" + args: "**/*.md" + ignore: ".github/ISSUE_TEMPLATE/*.md docs/usage/liqoctl/*.md" doclint: name: Lint documentation files runs-on: ubuntu-latest steps: - - name: Check out code - uses: actions/checkout@v4 - with: - ref: "${{ github.event.pull_request.head.sha }}" - repository: ${{github.event.pull_request.head.repo.full_name}} - persist-credentials: false - - name: Setup python3 - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - name: Install python3 dependencies - working-directory: ./docs - run: | - pip3 install -r requirements.txt - - name: Build documentation - working-directory: ./docs - run: | - make dummy - - name: Check external links - working-directory: ./docs - run: | - make linkcheck + - name: Check out code + uses: actions/checkout@v4 + with: + ref: "${{ github.event.pull_request.head.sha }}" + repository: ${{github.event.pull_request.head.repo.full_name}} + persist-credentials: false + - name: Setup python3 + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install python3 dependencies + working-directory: ./docs + run: | + pip3 install -r requirements.txt + - name: Build documentation + working-directory: ./docs + run: | + make dummy helmlint: name: Lint helm chart diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml deleted file mode 100644 index dbfa9957fa..0000000000 --- a/.github/workflows/merge.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Merge the current PR -on: - repository_dispatch: - types: - - merge-command - -jobs: - automerge: - name: Merge - runs-on: ubuntu-latest - - steps: - - name: Add the merge-requested label - uses: actions-ecosystem/action-add-labels@v1 - with: - github_token: "${{ secrets.CI_TOKEN }}" - number: ${{ github.event.client_payload.github.payload.issue.number }} - labels: merge-requested - - - name: Automerge - uses: pascalgn/automerge-action@v0.16.4 - env: - GITHUB_TOKEN: "${{ secrets.CI_TOKEN }}" - MERGE_LABELS: "merge-requested,!hold" - MERGE_METHOD: "rebase" - MERGE_RETRY_SLEEP: "30000" # 30 seconds - # Use a fake UPDATE_LABEL, to prevent rebase from being triggered - # https://github.com/pascalgn/automerge-action/issues/86 - UPDATE_LABELS: "never-perform-rebase,!hold" - UPDATE_METHOD: "rebase" - - - name: Remove the merge-requested label - uses: actions-ecosystem/action-remove-labels@v1 - with: - github_token: "${{ secrets.CI_TOKEN }}" - number: ${{ github.event.client_payload.github.payload.issue.number }} - labels: merge-requested diff --git a/.github/workflows/slash-commands.yml b/.github/workflows/slash-commands.yml deleted file mode 100644 index 59abdb6d7c..0000000000 --- a/.github/workflows/slash-commands.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Dispatch Slash Commands -on: - issue_comment: - types: - - created - -jobs: - dispatch: - name: Dispatch - runs-on: ubuntu-latest - - steps: - - name: Dispatch Slash Commands - uses: peter-evans/slash-command-dispatch@v4.0.0 - with: - token: ${{ secrets.CI_TOKEN }} - reaction-token: ${{ secrets.CI_TOKEN }} - issue-type: pull-request - config: > - [ - { - "command": "merge", - "permission": "write" - }, - { - "command": "rebase", - "permission": "triage" - }, - { - "command": "test", - "permission": "triage" - }, - { - "command": "build", - "permission": "triage" - }, - { - "command": "hold", - "permission": "none" - }, - { - "command": "unhold", - "permission": "none" - } - ] diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fb2c61c416..2511f5f933 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,79 +1,25 @@ name: Unit Testing on: - repository_dispatch: + push: + branches: + - "master" + pull_request: + issue_comment: types: - - test-command + - created jobs: - configure: - name: Preliminary configuration - runs-on: ubuntu-latest - outputs: - commit-ref: ${{ steps.configure.outputs.commit-ref }} - repo-suffix: ${{ steps.configure.outputs.repo-suffix }} - repo-name: ${{ steps.configure.outputs.repo-name }} - steps: - - name: Get the version - id: get_version - run: echo "VERSION=$(echo "$GITHUB_REF" | cut -d / -f 3)" >> "$GITHUB_OUTPUT" - if: startsWith(github.ref, 'refs/tags/v') - - name: Configure - id: configure - run: | - # The ref of the commit to checkout (do not use the merge commit if pull request) - if [ "${{ github.event_name }}" == "repository_dispatch" ]; then - echo "commit-ref=${{ github.event.client_payload.pull_request.head.sha }}" >> "$GITHUB_OUTPUT" - elif [ "${{ steps.get_version.outputs.VERSION }}" != "" ]; then - echo "commit-ref=${{ steps.get_version.outputs.VERSION }}" >> "$GITHUB_OUTPUT" - else - echo "commit-ref=${{ github.sha }}" >> "$GITHUB_OUTPUT" - fi - - if [ "${{ github.event_name }}" != "repository_dispatch" ]; then - echo "repo-name=${{ github.repository }}" >> "$GITHUB_OUTPUT" - else - echo "repo-name=${{ github.event.client_payload.github.payload.repository.full_name }}" >> "$GITHUB_OUTPUT" - fi - - - # Since we are using a repository-dispatch event, we have to explicitly set a run check. We initialize it to a "pending" state. - - uses: octokit/request-action@v2.x - name: "Initialize run check to 'pending'" - with: - route: POST /repos/${{ github.repository }}/statuses/${{ steps.configure.outputs.commit-ref }} - state: "pending" - description: "Unit Test status" - context: "Unit Test" - target_url: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - env: - GITHUB_TOKEN: ${{ secrets.CI_TOKEN }} - if: ${{ github.event_name == 'repository_dispatch' }} - test: name: Launch Test runs-on: ubuntu-latest - needs: configure + if: github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'issue_comment' && contains(github.event.comment.body, '/test')) steps: - - - name: Check out code into the Go module directory - uses: actions/checkout@v4 - with: - ref: "${{ needs.configure.outputs.commit-ref }}" - repository: "${{ needs.configure.outputs.repo-name }}" - persist-credentials: false - - - name: Launch Test (Docker Container) - run: make unit - - # Since we are using a repository-dispatch event, we have to explicitly set a run check. We update it to the actual status. - - uses: octokit/request-action@v2.x - name: "Update run check status" - with: - route: POST /repos/${{ github.repository }}/statuses/${{ needs.configure.outputs.commit-ref }} - state: "${{ job.status }}" - description: "Unit Test status" - context: "Unit Test" - target_url: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - env: - GITHUB_TOKEN: ${{ secrets.CI_TOKEN }} - if: ${{ !cancelled() && github.event_name == 'repository_dispatch' }} + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + persist-credentials: false + + - name: Launch Test (Docker Container) + run: make unit diff --git a/.gitignore b/.gitignore index 7dc4e7606f..f9b67c385a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ bin # Test binary, build with `go test -c` *.test +.kimchi # Output of the go coverage tool, specifically when used with LiteIDE *.out @@ -50,6 +51,7 @@ docs/_build /tmp /graphviz /k3s-ansible +.venv # AI tools .claude diff --git a/.krew.yaml b/.krew.yaml index fe9d8fc8ee..15617944ec 100644 --- a/.krew.yaml +++ b/.krew.yaml @@ -4,7 +4,7 @@ metadata: name: liqo spec: version: {{ .TagName }} - homepage: https://github.com/liqotech/liqo + homepage: https://github.com/castai/liqo shortDescription: Install and manage Liqo on your clusters description: | Liqo is a platform to enable dynamic and decentralized resource sharing across @@ -20,29 +20,29 @@ spec: matchLabels: os: darwin arch: amd64 - {{addURIAndSha "https://github.com/liqotech/liqo/releases/download/{{ .TagName }}/liqoctl-darwin-amd64.tar.gz" .TagName }} + {{addURIAndSha "https://github.com/castai/liqo/releases/download/{{ .TagName }}/liqoctl-darwin-amd64.tar.gz" .TagName }} bin: liqoctl - selector: matchLabels: os: darwin arch: arm64 - {{addURIAndSha "https://github.com/liqotech/liqo/releases/download/{{ .TagName }}/liqoctl-darwin-arm64.tar.gz" .TagName }} + {{addURIAndSha "https://github.com/castai/liqo/releases/download/{{ .TagName }}/liqoctl-darwin-arm64.tar.gz" .TagName }} bin: liqoctl - selector: matchLabels: os: linux arch: amd64 - {{addURIAndSha "https://github.com/liqotech/liqo/releases/download/{{ .TagName }}/liqoctl-linux-amd64.tar.gz" .TagName }} + {{addURIAndSha "https://github.com/castai/liqo/releases/download/{{ .TagName }}/liqoctl-linux-amd64.tar.gz" .TagName }} bin: liqoctl - selector: matchLabels: os: linux arch: arm64 - {{addURIAndSha "https://github.com/liqotech/liqo/releases/download/{{ .TagName }}/liqoctl-linux-arm64.tar.gz" .TagName }} + {{addURIAndSha "https://github.com/castai/liqo/releases/download/{{ .TagName }}/liqoctl-linux-arm64.tar.gz" .TagName }} bin: liqoctl - selector: matchLabels: os: windows arch: amd64 - {{addURIAndSha "https://github.com/liqotech/liqo/releases/download/{{ .TagName }}/liqoctl-windows-amd64.tar.gz" .TagName }} + {{addURIAndSha "https://github.com/castai/liqo/releases/download/{{ .TagName }}/liqoctl-windows-amd64.tar.gz" .TagName }} bin: liqoctl diff --git a/apis/networking/v1beta1/firewall/filterrule_types.go b/apis/networking/v1beta1/firewall/filterrule_types.go index 0e6e806b4b..6c86d4899e 100644 --- a/apis/networking/v1beta1/firewall/filterrule_types.go +++ b/apis/networking/v1beta1/firewall/filterrule_types.go @@ -39,6 +39,11 @@ const ( // ActionNotrack is the action to be applied to the rule. // ActionNotrack disables connection tracking for the matched packet. ActionNotrack FilterAction = "notrack" + // ActionSetMetaMark is the action to be applied to the rule. + // It is used to set the packet meta mark (fwmark) directly to the value specified in Value. + // Unlike ActionCtMark + ActionSetMetaMarkFromCtMark, this sets the mark per-packet + // (not per-connection) and is available immediately for route lookup when used in prerouting. + ActionSetMetaMark FilterAction = "setmetamark" ) // FilterRule is a rule to be applied to a filter chain. @@ -53,7 +58,7 @@ type FilterRule struct { // They can be multiple and they are applied with an AND operator. Match []Match `json:"match"` // Action is the action to be applied to the rule. - // +kubebuilder:validation:Enum=ctmark;metamarkfromctmark;tcpmssclamp;accept;drop;reject;notrack + // +kubebuilder:validation:Enum=ctmark;metamarkfromctmark;tcpmssclamp;accept;drop;reject;notrack;setmetamark Action FilterAction `json:"action"` // Value is the value to be used for the action. Value *string `json:"value,omitempty"` diff --git a/apis/networking/v1beta1/firewall/match_types.go b/apis/networking/v1beta1/firewall/match_types.go index 4d0c7eb7b1..0eba289c3c 100644 --- a/apis/networking/v1beta1/firewall/match_types.go +++ b/apis/networking/v1beta1/firewall/match_types.go @@ -82,6 +82,11 @@ type MatchDev struct { // Position is the source device of the packet. // +kubebuilder:validation:Enum=in;out Position MatchDevPosition `json:"position"` + // Wildcard enables prefix matching on the device name. + // When true, the match compares only the prefix bytes of Value against the interface name, + // allowing a single rule to match all interfaces sharing a common prefix (e.g. "liqo."). + // +kubebuilder:default=false + Wildcard bool `json:"wildcard,omitempty"` } // MatchProto is a protocol to be matched. diff --git a/apis/networking/v1beta1/internalfabric_types.go b/apis/networking/v1beta1/internalfabric_types.go index 23cc98c8b7..510d49ce84 100644 --- a/apis/networking/v1beta1/internalfabric_types.go +++ b/apis/networking/v1beta1/internalfabric_types.go @@ -67,8 +67,17 @@ type InternalFabricSpec struct { GatewayIP IP `json:"gatewayIP"` } +// InternalFabricStatus defines the observed state of InternalFabric. +type InternalFabricStatus struct { + // ECMPMark is the policy-routing fwmark assigned to this gateway for ECMP consistency. + // It is allocated by the controller and persisted here so it remains stable across reconnects. + // +optional + ECMPMark *int `json:"ecmpMark,omitempty"` +} + // +kubebuilder:object:root=true // +kubebuilder:resource:categories=liqo,shortName=if;ifabric +// +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Gateway IP",type=string,JSONPath=`.spec.gatewayIP` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` @@ -79,7 +88,8 @@ type InternalFabric struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec InternalFabricSpec `json:"spec,omitempty"` + Spec InternalFabricSpec `json:"spec,omitempty"` + Status InternalFabricStatus `json:"status,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/networking/v1beta1/zz_generated.deepcopy.go b/apis/networking/v1beta1/zz_generated.deepcopy.go index 24e1d501bc..e34f30ef30 100644 --- a/apis/networking/v1beta1/zz_generated.deepcopy.go +++ b/apis/networking/v1beta1/zz_generated.deepcopy.go @@ -813,6 +813,7 @@ func (in *InternalFabric) DeepCopyInto(out *InternalFabric) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InternalFabric. @@ -933,6 +934,26 @@ func (in *InternalFabricSpecInterfaceNode) DeepCopy() *InternalFabricSpecInterfa return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InternalFabricStatus) DeepCopyInto(out *InternalFabricStatus) { + *out = *in + if in.ECMPMark != nil { + in, out := &in.ECMPMark, &out.ECMPMark + *out = new(int) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InternalFabricStatus. +func (in *InternalFabricStatus) DeepCopy() *InternalFabricStatus { + if in == nil { + return nil + } + out := new(InternalFabricStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InternalGatewayEndpoint) DeepCopyInto(out *InternalGatewayEndpoint) { *out = *in diff --git a/apis/offloading/v1beta1/vkoptionstemplate_types.go b/apis/offloading/v1beta1/vkoptionstemplate_types.go index 3e3c5a0388..d84721e694 100644 --- a/apis/offloading/v1beta1/vkoptionstemplate_types.go +++ b/apis/offloading/v1beta1/vkoptionstemplate_types.go @@ -39,6 +39,8 @@ type VkOptionsTemplateSpec struct { ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` PullPolicy corev1.PullPolicy `json:"pullPolicy,omitempty"` Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` } // ReflectorConfig contains configuration parameters of the reflector. diff --git a/apis/offloading/v1beta1/zz_generated.deepcopy.go b/apis/offloading/v1beta1/zz_generated.deepcopy.go index d512386fe2..a5a939f9d2 100644 --- a/apis/offloading/v1beta1/zz_generated.deepcopy.go +++ b/apis/offloading/v1beta1/zz_generated.deepcopy.go @@ -997,6 +997,18 @@ func (in *VkOptionsTemplateSpec) DeepCopyInto(out *VkOptionsTemplateSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(v1.Affinity) + (*in).DeepCopyInto(*out) + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VkOptionsTemplateSpec. diff --git a/build/liqo/build.sh b/build/liqo/build.sh index b90caef143..f551bea8a2 100755 --- a/build/liqo/build.sh +++ b/build/liqo/build.sh @@ -4,8 +4,14 @@ set -e set -o nounset set -o pipefail +usage() { + echo "Usage: $0 [-m] [-p] " + echo " -p Push the built image to the registry" + echo " -h Show this help message" +} + if [ $# -ne 1 ]; then - echo "Usage: $0 " + usage exit 1 fi @@ -18,12 +24,17 @@ fi component=$(basename "$componentdir") # Set registry/org/tag if not already defined -DOCKER_REGISTRY="${DOCKER_REGISTRY:-ghcr.io}" -DOCKER_ORGANIZATION="${DOCKER_ORGANIZATION:-liqotech}" +DOCKER_REGISTRY="${DOCKER_REGISTRY:-us-docker.pkg.dev}" +DOCKER_ORGANIZATION="${DOCKER_ORGANIZATION:-castai-hub/library/liqo}" DOCKER_TAG="${DOCKER_TAG:-latest}" DOCKER_PUSH="${DOCKER_PUSH:-true}" ARCHS="${ARCHS:-linux/amd64,linux/arm64}" +# Secondary registry: when both are set, the image is also tagged +# (and pushed) as ${GHCR_REGISTRY}/${GHCR_IMAGE_PREFIX}/ +GHCR_REGISTRY="${GHCR_REGISTRY:-}" +GHCR_IMAGE_PREFIX="${GHCR_IMAGE_PREFIX:-}" + echo "Downloading Go modules..." start_time=$(date +%s) go mod download @@ -66,12 +77,22 @@ else fi if [[ "$DOCKER_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then - image_tag="${DOCKER_REGISTRY}/$DOCKER_ORGANIZATION/${image_component}:${DOCKER_TAG}" + repo_suffix="" else - image_tag="${DOCKER_REGISTRY}/${DOCKER_ORGANIZATION}/${image_component}-ci:${DOCKER_TAG}" + repo_suffix="-ci" +fi + +image_tag="${DOCKER_REGISTRY}/${DOCKER_ORGANIZATION}/${image_component}${repo_suffix}:${DOCKER_TAG}" +tag_args=(-t "$image_tag") + +if [ -n "$GHCR_REGISTRY" ] && [ -n "$GHCR_IMAGE_PREFIX" ]; then + ghcr_tag="${GHCR_REGISTRY}/${GHCR_IMAGE_PREFIX}/${image_component}${repo_suffix}:${DOCKER_TAG}" + tag_args+=(-t "$ghcr_tag") + echo "Also tagging container image as $ghcr_tag" fi + echo "Building container image $image_tag for architectures $ARCHS..." docker buildx build --platform "${ARCHS}" \ --build-arg COMPONENT="$component" \ - -t "$image_tag" -f ./build/liqo/Dockerfile . \ + "${tag_args[@]}" -f ./build/liqo/Dockerfile . \ "$(if $DOCKER_PUSH; then echo --push; else echo --load; fi)" diff --git a/cmd/fabric/main.go b/cmd/fabric/main.go index 349ce0cff6..2a311efe77 100644 --- a/cmd/fabric/main.go +++ b/cmd/fabric/main.go @@ -42,12 +42,12 @@ import ( sourcedetector "github.com/liqotech/liqo/pkg/fabric/source-detector" "github.com/liqotech/liqo/pkg/firewall" "github.com/liqotech/liqo/pkg/gateway" - "github.com/liqotech/liqo/pkg/gateway/concurrent" "github.com/liqotech/liqo/pkg/gateway/tunnel" "github.com/liqotech/liqo/pkg/liqo-controller-manager/networking/external-network/remapping" "github.com/liqotech/liqo/pkg/route" argsutils "github.com/liqotech/liqo/pkg/utils/args" flagsutils "github.com/liqotech/liqo/pkg/utils/flags" + "github.com/liqotech/liqo/pkg/utils/kernel" kernelversion "github.com/liqotech/liqo/pkg/utils/kernel/version" "github.com/liqotech/liqo/pkg/utils/mapper" "github.com/liqotech/liqo/pkg/utils/resource" @@ -102,6 +102,12 @@ func run(cmd *cobra.Command, _ []string) error { } } + if options.EnableMultipathHashPolicy { + if err = kernel.EnableMultipathHashPolicy(); err != nil { + return fmt.Errorf("failed to enable multipath hash policy: %w", err) + } + } + // Set controller-runtime logger. log.SetLogger(klog.NewKlogr()) @@ -119,12 +125,6 @@ func run(cmd *cobra.Command, _ []string) error { []string{gateway.GatewayComponentGateway}, ) utilruntime.Must(err) - reqActiveGatewayPods, err := labels.NewRequirement( - concurrent.ActiveGatewayKey, - selection.Equals, - []string{concurrent.ActiveGatewayValue}, - ) - utilruntime.Must(err) // Create the manager. mgr, err := ctrl.NewManager(cfg, ctrl.Options{ @@ -138,7 +138,7 @@ func run(cmd *cobra.Command, _ []string) error { NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) { opts.ByObject = map[client.Object]cache.ByObject{ &corev1.Pod{}: { - Label: labels.NewSelector().Add(*reqGatewayPods).Add(*reqActiveGatewayPods), + Label: labels.NewSelector().Add(*reqGatewayPods), }, } return cache.New(config, opts) diff --git a/cmd/gateway/geneve/main.go b/cmd/gateway/geneve/main.go index d86784004b..912821eaf0 100644 --- a/cmd/gateway/geneve/main.go +++ b/cmd/gateway/geneve/main.go @@ -32,7 +32,6 @@ import ( networkingv1beta1 "github.com/liqotech/liqo/apis/networking/v1beta1" "github.com/liqotech/liqo/pkg/conncheck" "github.com/liqotech/liqo/pkg/gateway" - "github.com/liqotech/liqo/pkg/gateway/concurrent" gwfabric "github.com/liqotech/liqo/pkg/gateway/fabric" flagsutils "github.com/liqotech/liqo/pkg/utils/flags" "github.com/liqotech/liqo/pkg/utils/mapper" @@ -48,7 +47,6 @@ func init() { utilruntime.Must(networkingv1beta1.AddToScheme(scheme)) } -// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;create;update;delete // +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete func main() { @@ -119,17 +117,6 @@ func run(cmd *cobra.Command, _ []string) error { return fmt.Errorf("unable to setup geneve tunnel reconciler: %w", err) } - if options.GwOptions.LeaderElection { - runnableGuest, err := concurrent.NewRunnableGuest(options.GwOptions.ContainerName) - if err != nil { - return fmt.Errorf("unable to create runnable guest: %w", err) - } - if err := runnableGuest.Start(cmd.Context()); err != nil { - return fmt.Errorf("unable to start runnable guest: %w", err) - } - defer runnableGuest.Close() - } - runnableGeneveCleanup, err := gwfabric.NewRunnableGeneveCleanup(mgr.GetClient(), options.GeneveCleanupInterval) if err != nil { return fmt.Errorf("unable to create runnable geneve cleanup: %w", err) diff --git a/cmd/gateway/main.go b/cmd/gateway/main.go index 88542dd017..b2d4a8ebf5 100644 --- a/cmd/gateway/main.go +++ b/cmd/gateway/main.go @@ -24,10 +24,8 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log" @@ -38,7 +36,6 @@ import ( "github.com/liqotech/liqo/pkg/conncheck" "github.com/liqotech/liqo/pkg/firewall" "github.com/liqotech/liqo/pkg/gateway" - "github.com/liqotech/liqo/pkg/gateway/concurrent" "github.com/liqotech/liqo/pkg/gateway/connection" "github.com/liqotech/liqo/pkg/gateway/tunnel" "github.com/liqotech/liqo/pkg/liqo-controller-manager/networking/external-network/remapping" @@ -64,9 +61,6 @@ func init() { utilruntime.Must(networkingv1beta1.AddToScheme(scheme)) } -// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;create;update;delete -// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;update;patch - func main() { var cmd = cobra.Command{ Use: "liqo-gateway", @@ -136,15 +130,6 @@ func run(cmd *cobra.Command, _ []string) error { // Get the rest config. cfg := config.GetConfigOrDie() - // Create the client. This client should be used only outside the reconciler. - // This client does not need a cache. - cl, err := client.New(cfg, client.Options{ - Scheme: scheme, - }) - if err != nil { - return fmt.Errorf("unable to create client: %w", err) - } - // Create the manager. mgr, err := ctrl.NewManager(cfg, ctrl.Options{ MapperProvider: mapper.LiqoMapperProvider(scheme), @@ -154,17 +139,6 @@ func run(cmd *cobra.Command, _ []string) error { }, HealthProbeBindAddress: connoptions.GwOptions.ProbeAddr, PprofBindAddress: connoptions.GwOptions.PprofAddr, - LeaderElection: connoptions.GwOptions.LeaderElection, - LeaderElectionID: fmt.Sprintf( - "%s.%s.%s.connections.liqo.io", - connoptions.GwOptions.Name, connoptions.GwOptions.Namespace, connoptions.GwOptions.Mode, - ), - LeaderElectionNamespace: connoptions.GwOptions.Namespace, - LeaderElectionReleaseOnCancel: true, - LeaderElectionResourceLock: resourcelock.LeasesResourceLock, - LeaseDuration: &connoptions.GwOptions.LeaderElectionLeaseDuration, - RenewDeadline: &connoptions.GwOptions.LeaderElectionRenewDeadline, - RetryPeriod: &connoptions.GwOptions.LeaderElectionRetryPeriod, }) if err != nil { return fmt.Errorf("unable to create manager: %w", err) @@ -245,23 +219,6 @@ func run(cmd *cobra.Command, _ []string) error { return fmt.Errorf("unable to setup firewall configuration reconciler: %w", err) } - if connoptions.GwOptions.LeaderElection { - runnable, err := concurrent.NewRunnableGatewayStartup( - cl, - connoptions.GwOptions.PodName, - connoptions.GwOptions.Name, - connoptions.GwOptions.Namespace, - connoptions.GwOptions.ConcurrentContainersNames, - ) - if err != nil { - return fmt.Errorf("unable to create concurrent runnable: %w", err) - } - - if err := mgr.Add(runnable); err != nil { - return fmt.Errorf("unable to add concurrent runnable: %w", err) - } - } - // Start the manager. return mgr.Start(cmd.Context()) } diff --git a/cmd/gateway/wireguard/main.go b/cmd/gateway/wireguard/main.go index 97a085ab95..8bff5495ad 100644 --- a/cmd/gateway/wireguard/main.go +++ b/cmd/gateway/wireguard/main.go @@ -37,7 +37,6 @@ import ( ipamv1alpha1 "github.com/liqotech/liqo/apis/ipam/v1alpha1" networkingv1beta1 "github.com/liqotech/liqo/apis/networking/v1beta1" "github.com/liqotech/liqo/pkg/gateway" - "github.com/liqotech/liqo/pkg/gateway/concurrent" "github.com/liqotech/liqo/pkg/gateway/tunnel/wireguard" flagsutils "github.com/liqotech/liqo/pkg/utils/flags" "github.com/liqotech/liqo/pkg/utils/mapper" @@ -55,7 +54,6 @@ func init() { utilruntime.Must(ipamv1alpha1.AddToScheme(scheme)) } -// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;create;update;delete // +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete func main() { @@ -102,7 +100,6 @@ func run(cmd *cobra.Command, _ []string) error { BindAddress: options.GwOptions.MetricsAddress, }, HealthProbeBindAddress: options.GwOptions.ProbeAddr, - LeaderElection: false, }) if err != nil { return fmt.Errorf("unable to create manager: %w", err) @@ -164,6 +161,7 @@ func run(cmd *cobra.Command, _ []string) error { // Create the Prometheus collector and register it inside the controller-runtime metrics server. promcollect, err := wireguard.NewPrometheusCollector(&wireguard.MetricsOptions{ RemoteClusterID: options.GwOptions.RemoteClusterID, + GatewayName: options.GwOptions.Name, Namespace: options.GwOptions.Namespace, WgImplementation: options.Implementation, }) @@ -174,17 +172,6 @@ func run(cmd *cobra.Command, _ []string) error { return fmt.Errorf("unable to register prometheus collector: %w", err) } - if options.GwOptions.LeaderElection { - runnable, err := concurrent.NewRunnableGuest(options.GwOptions.ContainerName) - if err != nil { - return fmt.Errorf("unable to create runnable guest: %w", err) - } - if err := runnable.Start(cmd.Context()); err != nil { - return fmt.Errorf("unable to start runnable guest: %w", err) - } - defer runnable.Close() - } - // Start the manager. return mgr.Start(cmd.Context()) } diff --git a/cmd/liqo-controller-manager/main.go b/cmd/liqo-controller-manager/main.go index d38782d41a..e62bad3326 100644 --- a/cmd/liqo-controller-manager/main.go +++ b/cmd/liqo-controller-manager/main.go @@ -258,15 +258,6 @@ func run(cmd *cobra.Command, _ []string) error { return fmt.Errorf("unable to start the offloadedPod reconciler: %w", err) } - configurationReconciler := ipmapping.NewConfigurationReconciler( - mgr.GetClient(), - mgr.GetScheme(), - mgr.GetEventRecorderFor("configuration-controller"), - ) - if err := configurationReconciler.SetupWithManager(mgr); err != nil { - return fmt.Errorf("unable to start the configuration reconciler: %w", err) - } - if !opts.APIServerIPUseEndpointSlices { if err := ipamips.EnforceAPIServerIPRemapping(cmd.Context(), uncachedClient, opts.LiqoNamespace); err != nil { return fmt.Errorf("unable to enforce the API server IP remapping: %w", err) diff --git a/cmd/liqoctl/cmd/network.go b/cmd/liqoctl/cmd/network.go index ad533bfe5d..8d9b326b7a 100644 --- a/cmd/liqoctl/cmd/network.go +++ b/cmd/liqoctl/cmd/network.go @@ -158,6 +158,7 @@ func newNetworkConnectCommand(ctx context.Context, options *network.Options) *co // Common flags cmd.Flags().IntVar(&options.MTU, "mtu", forge.DefaultMTU, fmt.Sprintf("MTU of the Gateway server and client. Default: %d", forge.DefaultMTU)) + cmd.Flags().Int32Var(&options.Replicas, "gw-replicas", 1, "Number of active-active gateway replicas to create") cmd.Flags().BoolVar(&options.DisableSharingKeys, "disable-sharing-keys", false, "Disable the sharing of public keys between the two clusters") runtime.Must(cmd.RegisterFlagCompletionFunc("gw-server-service-type", completion.Enumeration(options.ServerServiceType.Allowed))) diff --git a/cmd/liqoctl/cmd/peer.go b/cmd/liqoctl/cmd/peer.go index 39f4830011..b87f03ac80 100644 --- a/cmd/liqoctl/cmd/peer.go +++ b/cmd/liqoctl/cmd/peer.go @@ -115,6 +115,7 @@ func newPeerCommand(ctx context.Context, f *factory.Factory) *cobra.Command { "not directly reachable (e.g. the server is behind a NAT)") cmd.Flags().IntVar(&options.MTU, "mtu", nwforge.DefaultMTU, fmt.Sprintf("MTU of the Gateway server and client. Default: %d", nwforge.DefaultMTU)) + cmd.Flags().Int32Var(&options.GwReplicas, "gw-replicas", 1, "Number of active-active gateway replicas to create") runtime.Must(cmd.RegisterFlagCompletionFunc("gw-server-service-location", completion.Enumeration(options.ServerServiceLocation.Allowed))) runtime.Must(cmd.RegisterFlagCompletionFunc("gw-server-service-type", completion.Enumeration(options.ServerServiceType.Allowed))) diff --git a/cmd/liqoctl/cmd/test.go b/cmd/liqoctl/cmd/test.go index 675a11e55a..dd2d6e1ed5 100644 --- a/cmd/liqoctl/cmd/test.go +++ b/cmd/liqoctl/cmd/test.go @@ -31,7 +31,9 @@ import ( const liqoctlTestNetworkLongHelp = `Launch network E2E tests. -WARNING: to run the tests you need to have kyverno installed on every cluster https://kyverno.io/docs/installation/methods/ . +WARNING: to run the tests you need to have Kyverno v1.13 or later installed on every cluster +(https://kyverno.io/docs/installation/methods/). The tests rely on the policies.kyverno.io/v1 +NamespacedMutatingPolicy API, which is not available in earlier Kyverno versions. This command allows to launch E2E tests, which are used to check the network functionalities between the clusters. The command needs to be run on the cluster that will act as the consumer, @@ -43,6 +45,8 @@ Examples: $ {{ .Executable }} test network --remote-kubeconfigs $HOME/.kube/config2,$HOME/.kube/config3 or $ {{ .Executable }} test network --remote-kubeconfigs $HOME/.kube/config2,$HOME/.kube/config3 --basic +or + $ {{ .Executable }} test network --remote-kubeconfigs $HOME/.kube/config2,$HOME/.kube/config3 --node-pod or $ {{ .Executable }} test network --remote-kubeconfigs $HOME/.kube/config2,$HOME/.kube/config3 --np-nodes all --np-ext --pod-np or diff --git a/cmd/virtual-kubelet/root/opts.go b/cmd/virtual-kubelet/root/opts.go index c36c6a8df8..9778bb58dd 100644 --- a/cmd/virtual-kubelet/root/opts.go +++ b/cmd/virtual-kubelet/root/opts.go @@ -56,7 +56,10 @@ var DefaultReflectorsWorkers = map[resources.ResourceReflected]uint{ resources.Secret: 3, resources.ServiceAccount: 3, resources.PersistentVolumeClaim: 3, + resources.LocalStoragePVC: 1, resources.Event: 3, + resources.ResourceSlice: 1, + resources.ResourceClaim: 3, } // DefaultReflectorsTypes contains the default type of reflection for each reflected resource. @@ -68,7 +71,10 @@ var DefaultReflectorsTypes = map[resources.ResourceReflected]offloadingv1beta1.R resources.Secret: offloadingv1beta1.DenyList, resources.ServiceAccount: offloadingv1beta1.CustomLiqo, resources.PersistentVolumeClaim: offloadingv1beta1.CustomLiqo, + resources.LocalStoragePVC: offloadingv1beta1.DenyList, resources.Event: offloadingv1beta1.DenyList, + resources.ResourceSlice: offloadingv1beta1.CustomLiqo, + resources.ResourceClaim: offloadingv1beta1.CustomLiqo, } // Opts stores all the options for configuring the root virtual-kubelet command. diff --git a/cmd/virtual-kubelet/root/root.go b/cmd/virtual-kubelet/root/root.go index aaf078dc8e..3df30ba031 100644 --- a/cmd/virtual-kubelet/root/root.go +++ b/cmd/virtual-kubelet/root/root.go @@ -272,6 +272,28 @@ func runRootCommand(ctx context.Context, c *Opts) error { if c.CreateNode { var nodeReady chan struct{} var nodeRunner *node.NodeController + + remoteCl, err := client.New(remoteConfig, client.Options{Scheme: scheme}) + if err != nil { + return fmt.Errorf("creating remote client: %w", err) + } + var remoteNode corev1.Node + var remoteNodePtr *corev1.Node + err = remoteCl.Get(ctx, client.ObjectKey{Name: c.NodeName}, &remoteNode) // local node name matches with remote node name + if err != nil && !k8serrors.IsForbidden(err) { + return fmt.Errorf("getting remote node: %w", err) + } + + // If the error is forbidden, it means that the virtual kubelet does not have permissions to get the remote node. + // This can happen if remote cluster is running a older liqo version. In such cases, we keep legacy behavior (defaulting + // node info fields) and we do not watch the remote node for updates. + if k8serrors.IsForbidden(err) { + klog.Warning("Unable to get remote node due to missing permissions; defaulting node info fields and skipping remote node watching") + } + if err == nil { + remoteNodePtr = &remoteNode + } + // Initialize the node provider nodecfg := nodeprovider.InitConfig{ HomeConfig: localConfig, @@ -293,6 +315,7 @@ func runRootCommand(ctx context.Context, c *Opts) error { VirtualNode: &vn, ForeignCluster: foreignCluster, + RemoteNode: remoteNodePtr, } nodeProvider := nodeprovider.NewLiqoNodeProvider(&nodecfg) @@ -307,27 +330,49 @@ func runRootCommand(ctx context.Context, c *Opts) error { func(ctx context.Context, _ error) error { klog.Info("node setting up") newNode := nodeProvider.GetNode().DeepCopy() + + klog.Infof("Restart check") + if newNode == nil || newNode.Name == "" { + klog.Errorf("Restartining the pod") + if newNode == nil { + klog.Errorf("newNode is nil") + } else if newNode.Name == "" { + klog.Errorf("newNode.Name is empty") + } + klog.Flush() // Force flush before exit + os.Exit(1) + } + newNode.ResourceVersion = "" if nodeProvider.IsTerminating() { // this avoids the re-creation of terminated nodes - klog.V(4).Info("skipping: node is in terminating phase") + klog.Info("skipping: node is in terminating phase") return nil } + klog.Infof("attempting to get node: %s", newNode.Name) oldNode, newErr := localClient.CoreV1().Nodes().Get(ctx, newNode.Name, metav1.GetOptions{}) if newErr != nil { if !k8serrors.IsNotFound(newErr) { klog.Error(newErr, "node error") return newErr } + klog.Info("node not found, creating new node") _, newErr = localClient.CoreV1().Nodes().Create(ctx, newNode, metav1.CreateOptions{}) - klog.Info("new node created") + if newErr == nil { + klog.Info("new node created") + } else { + klog.Errorf("failed to create node: %v", newErr) + } } else { + klog.Info("node exists, updating status") oldNode.Status = newNode.Status _, newErr = localClient.CoreV1().Nodes().UpdateStatus(ctx, oldNode, metav1.UpdateOptions{}) - if newErr != nil { + if newErr == nil { klog.Info("node updated") + } else { + klog.Errorf("failed to update node: %v", newErr) } } @@ -374,7 +419,11 @@ func getVersion(config *rest.Config) string { } func isReflectionTypeNotCustomizable(resource resources.ResourceReflected) bool { - return resource == resources.Pod || resource == resources.ServiceAccount || resource == resources.PersistentVolumeClaim + return resource == resources.Pod || + resource == resources.ServiceAccount || + resource == resources.PersistentVolumeClaim || + resource == resources.ResourceSlice || + resource == resources.ResourceClaim } func getReflectorsConfigs(c *Opts) (map[resources.ResourceReflected]offloadingv1beta1.ReflectorConfig, error) { diff --git a/deployments/liqo/README.md b/deployments/liqo/README.md index c57235b3f5..9a1f70932b 100644 --- a/deployments/liqo/README.md +++ b/deployments/liqo/README.md @@ -22,7 +22,7 @@ | controllerManager.config.defaultLimitsEnforcement | string | `"None"` | Defines how strict is the enforcement of the quota offered by the remote cluster. enableResourceEnforcement must be enabled to use this feature. Possible values are: None, Soft, Hard. None: the offloaded pods might not have the resource `requests` or `limits`. Soft: it forces the offloaded pods to have `requests` set. If the pods go over the requests, the total used resources might go over the quota. Hard: it forces the offloaded pods to have `limits` and `requests` set, with `requests` == `limits`. This is the safest mode as the consumer cluster cannot go over the quota. | | controllerManager.config.enableNodeFailureController | bool | `false` | Ensure offloaded pods running on a failed node are evicted and rescheduled on a healthy node, preventing them to remain in a terminating state indefinitely. This feature can be useful in case of remote node failure to guarantee better service continuity and to have the expected pods workload on the remote cluster. However, enabling this feature could produce zombies in the worker node, in case the node returns Ready again without a restart. | | controllerManager.config.enableResourceEnforcement | bool | `true` | It enforces offerer-side that offloaded pods do not exceed offered resources (based on container limits). This feature is suggested to be enabled when consumer-side enforcement is not sufficient. It makes sure that the sum of the requests of the offloaded pods never exceeds the quota offered by the remote cluster. The quota can be still exceeded if no limits and requests are defined in the offloaded pods or if the limits are larger than the requests. For a stricter enforcement, the defaultLimitsEnforcement can be set to Hard. | -| controllerManager.image.name | string | `"ghcr.io/liqotech/liqo-controller-manager"` | Image repository for the controller-manager pod. | +| controllerManager.image.name | string | `"ghcr.io/castai/images/liqo/liqo-controller-manager"` | Image repository for the controller-manager pod. | | controllerManager.image.version | string | `""` | Custom version for the controller-manager image. If not specified, the global tag is used. | | controllerManager.metrics.service | object | `{"annotations":{},"labels":{}}` | Service used to expose metrics. | | controllerManager.metrics.service.annotations | object | `{}` | Annotations for the metrics service. | @@ -37,7 +37,7 @@ | controllerManager.pod.priorityClassName | string | `""` | PriorityClassName (https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority) for the controller-manager pod. | | controllerManager.pod.resources | object | `{"limits":{},"requests":{}}` | Resource requests and limits (https://kubernetes.io/docs/user-guide/compute-resources/) for the controller-manager pod. | | controllerManager.replicas | int | `1` | The number of controller-manager instances to run, which can be increased for active/passive high availability. | -| crdReplicator.image.name | string | `"ghcr.io/liqotech/crd-replicator"` | Image repository for the crdReplicator pod. | +| crdReplicator.image.name | string | `"ghcr.io/castai/images/liqo/crd-replicator"` | Image repository for the crdReplicator pod. | | crdReplicator.image.version | string | `""` | Custom version for the crdReplicator image. If not specified, the global tag is used. | | crdReplicator.metrics.podMonitor.enabled | bool | `false` | Enable/Disable the creation of a Prometheus podmonitor. Turn on this flag when the Prometheus Operator runs in your cluster | | crdReplicator.metrics.podMonitor.interval | string | `""` | Setup pod monitor requests interval. If empty, Prometheus uses the global scrape interval (https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint). | @@ -56,7 +56,7 @@ | ipam.external.url | string | `""` | The URL of the external IPAM. | | ipam.externalCIDR | string | `""` | The IP subnet used for the external CIDR. If empty, a free network will be automatically allocated by the IPAM. If set, the IPAM will try to allocate the exact network, failing in case of conflicts. Set it only if you know what you are doing. | | ipam.internal.graphviz | bool | `false` | Enable/Disable the generation of graphviz files inside the ipam. This feature is useful to visualize the status of the ipam. The graphviz files are stored in the /graphviz directory of the ipam pod (a file for each network pool). You can access them using "kubectl cp". | -| ipam.internal.image.name | string | `"ghcr.io/liqotech/ipam"` | Image repository for the IPAM pod. | +| ipam.internal.image.name | string | `"ghcr.io/castai/images/liqo/ipam"` | Image repository for the IPAM pod. | | ipam.internal.image.version | string | `""` | Custom version for the IPAM image. If not specified, the global tag is used. | | ipam.internal.pod.annotations | object | `{}` | Annotations for the IPAM pod. | | ipam.internal.pod.extraArgs | list | `[]` | Extra arguments for the IPAM pod. | @@ -72,18 +72,18 @@ | ipam.pools | list | `["10.0.0.0/8","192.168.0.0/16","172.16.0.0/12"]` | Set of network pools to perform the automatic address mapping in Liqo. Network pools are used to map a cluster network into another one in order to prevent conflicts. If left empty, it is defaulted to the private addresses ranges: [10.0.0.0/8, 192.168.0.0/16, 172.16.0.0/12] | | ipam.reservedSubnets | list | `[]` | List of IP subnets that do not have to be used by Liqo. Liqo can perform automatic IP address remapping when a remote cluster is peering with you, e.g., in case IP address spaces (e.g., PodCIDR) overlaps. In order to prevent IP conflicting between locally used private subnets in your infrastructure and private subnets belonging to remote clusters you need tell liqo the subnets used in your cluster. E.g if your cluster nodes belong to the 192.168.2.0/24 subnet, then you should add that subnet to the reservedSubnets. PodCIDR and serviceCIDR used in the local cluster are automatically added to the reserved list. | | ipam.serviceCIDR | string | `""` | The IP subnet used by the services in you cluster, in CIDR notation (e.g., 172.16.0.0/16). | -| liqo-crds | object | `{"crdUpgrade":{"enabled":false,"image":{"name":"ghcr.io/liqotech/liqo-crd-upgrade","pullPolicy":"IfNotPresent","version":""},"keepResources":false}}` | Liqo CRD subchart configuration Since Liqo contains a lot of CRDs, we decided to separate them from the main chart These values override the ones in the liqo-crds subchart | +| liqo-crds | object | `{"crdUpgrade":{"enabled":false,"image":{"name":"ghcr.io/castai/images/liqo/liqo-crd-upgrade","pullPolicy":"IfNotPresent","version":""},"keepResources":false}}` | Liqo CRD subchart configuration Since Liqo contains a lot of CRDs, we decided to separate them from the main chart These values override the ones in the liqo-crds subchart | | liqo-crds.crdUpgrade.enabled | bool | `false` | Enable/Disable the CRD upgrade job. When enabled, a pre-upgrade hook will automatically update CRDs during helm upgrade. | -| liqo-crds.crdUpgrade.image.name | string | `"ghcr.io/liqotech/liqo-crd-upgrade"` | Image repository for the CRD upgrade job. NOTE: This image only supports linux/amd64 and linux/arm64 platforms (arm/v7 is not supported). | +| liqo-crds.crdUpgrade.image.name | string | `"ghcr.io/castai/images/liqo/liqo-crd-upgrade"` | Image repository for the CRD upgrade job. NOTE: This image only supports linux/amd64 and linux/arm64 platforms (arm/v7 is not supported). | | liqo-crds.crdUpgrade.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy for the CRD upgrade job. | | liqo-crds.crdUpgrade.image.version | string | `""` | Image version for the CRD upgrade job. Required when crdUpgrade.enabled is true. | | liqo-crds.crdUpgrade.keepResources | bool | `false` | Keep the CRD upgrade resources (Job, ConfigMap, ServiceAccount, ClusterRole, ClusterRoleBinding) after completion. If false, they are deleted on success. | | metricAgent.config.clientSideRateLimit | object | `{"burst":500,"qps":500}` | Client-side rate limiting (QPS/burst) of the metric agent towards the local API server. The metric agent issues one request per physical node for each incoming scrape request, hence the defaults need to be high enough to avoid artificial throttling in clusters with many nodes. Setting qps to 0 disables the rate limiter entirely (use with caution). | | metricAgent.config.timeout | object | `{"read":"30s","write":"30s"}` | Set the timeout for the metrics server. | | metricAgent.enabled | bool | `true` | Enable/Disable the virtual kubelet metric agent. This component aggregates all the kubelet-related metrics (e.g., CPU, RAM, etc) collected on the nodes that are used by a remote cluster peered with you, then exporting the resulting values as a property of the virtual kubelet running on the remote cluster. | -| metricAgent.image.name | string | `"ghcr.io/liqotech/metric-agent"` | Image repository for the metricAgent pod. | +| metricAgent.image.name | string | `"ghcr.io/castai/images/liqo/metric-agent"` | Image repository for the metricAgent pod. | | metricAgent.image.version | string | `""` | Custom version for the metricAgent image. If not specified, the global tag is used. | -| metricAgent.initContainer.image.name | string | `"ghcr.io/liqotech/cert-creator"` | Image repository for the init container of the metricAgent pod. | +| metricAgent.initContainer.image.name | string | `"ghcr.io/castai/images/liqo/cert-creator"` | Image repository for the init container of the metricAgent pod. | | metricAgent.initContainer.image.version | string | `""` | Custom version for the init container image of the metricAgent pod. If not specified, the global tag is used. | | metricAgent.pod.annotations | object | `{}` | Annotations for the metricAgent pod. | | metricAgent.pod.extraArgs | list | `[]` | Extra arguments for the metricAgent pod. | @@ -102,6 +102,7 @@ | networking.fabric.config.gatewayMasqueradeBypass | bool | `false` | Enable/Disable the masquerade bypass for the gateway pods. It means that the packets from gateway pods will not be masqueraded from the host where the pod is scheduled. This is useful in scenarios where CNIs masquerade the traffic from pod to nodes. For example this is required when using the Azure CNI or Kindnet. | | networking.fabric.config.healthProbeBindAddressPort | string | `"8081"` | Set the port where the fabric pod will expose the health probe. To disable the health probe, set the port to 0. | | networking.fabric.config.metricsAddressPort | string | `"8082"` | Set the port where the fabric pod will expose the metrics. To disable the metrics, set the port to 0. | +| networking.fabric.config.multipathHashPolicy | bool | `true` | Enable 5-tuple ECMP hashing (fib_multipath_hash_policy=1) in the fabric pod. This improves load balancing when multiple gateway replicas provide ECMP next-hops. | | networking.fabric.config.nftablesMonitor | bool | `false` | Enable/Disable the nftables monitor for the fabric pod. It means that the fabric pod will monitor the nftables rules and will restore them in case of changes. In some cases (like K3S), this monitor can cause a huge amount of CPU usage. If you are experiencing high CPU usage, you can disable this feature. | | networking.fabric.config.ping | object | `{"interval":"10s","lossThreshold":5,"port":54321,"updateStatusInterval":"10s"}` | Set the options to configure the fabric ping used to check connection with the gateway through geneve | | networking.fabric.config.ping.interval | string | `"10s"` | Set the interval between two consecutive pings | @@ -110,7 +111,7 @@ | networking.fabric.config.ping.updateStatusInterval | string | `"10s"` | Set the interval at which the connection resource status is updated | | networking.fabric.config.routeConfigurationRulePriority | int | `0` | The priority used by the liqo-controller-manager to set the routing rules on the fabric (hostnetwork). It can be useful to solve race conditions between CNI daemonsets and liqo-fabric. For example the AWS CNI should add some rules with a fixed lower priority (lowest comes first) which breaks Liqo networking. Setting an explicit priority on Liqo's fabric routing rules allows them to be correctly ordered with respect to CNI rules. We suggest using a value between 101 and 500. If the value is 0, no priority is set, and Linux will automatically select a lower priority than the existing rules. | | networking.fabric.config.routeMonitor | bool | `true` | Enable/Disable the route monitor for the fabric pod. It means that the fabric pod will monitor the routing rules and will restore them in case of changes. | -| networking.fabric.image.name | string | `"ghcr.io/liqotech/fabric"` | Image repository for the fabric pod. | +| networking.fabric.image.name | string | `"ghcr.io/castai/images/liqo/fabric"` | Image repository for the fabric pod. | | networking.fabric.image.version | string | `""` | Custom version for the fabric image. If not specified, the global tag is used. | | networking.fabric.nodeSelector | object | `{}` | NodeSelector for the fabric pod. | | networking.fabric.pod.annotations | object | `{}` | Annotations for the fabric pod. | @@ -122,15 +123,15 @@ | networking.gateway.gatewayTemplateWatchEnabled | bool | `true` | Enable watching of custom GatewayTemplate CRDs. | | networking.gateway.mssclamp | object | `{"enabled":true,"value":0}` | Enable the TCP MSS clamping on tunnel interfaces. Tunneling technologies introduce extra overhead that reduces the MTU, causing standard-sized Internet packets to exceed the tunnel's capacity and be dropped. TCP MSS Clamping resolves this by intercepting the initial TCP connection handshake and dynamically rewriting the Maximum Segment Size (MSS) value to match the smaller available space of the tunnel interface. This dynamic adjustment, per TCP-session, forces the remote server to generate smaller data packets that fit inside the tunnel, effectively preventing fragmentation issues and the common "black hole" phenomenon where connections establish but data transfer hangs indefinitely. | | networking.gateway.mssclamp.value | int | `0` | Set the value for the mssclamp rule. Set to 0 to use automatic value discovery based on the MTU of the tunnel interface. | -| networking.gatewayTemplates | object | `{"container":{"gateway":{"extraArgs":[],"image":{"name":"ghcr.io/liqotech/gateway","version":""},"resources":{"limits":{},"requests":{}}},"geneve":{"image":{"name":"ghcr.io/liqotech/gateway/geneve","version":""},"resources":{"limits":{},"requests":{}}},"wireguard":{"image":{"name":"ghcr.io/liqotech/gateway/wireguard","version":""},"resources":{"limits":{},"requests":{}}}},"nftablesMonitor":true,"ping":{"interval":"2s","lossThreshold":5,"updateStatusInterval":"10s"},"pod":{"affinity":{},"annotations":{},"nodeSelector":{},"priorityClassName":"","tolerations":[]},"replicas":1,"routeMonitor":true,"server":{"service":{"allocateLoadBalancerNodePorts":"","annotations":{}}},"wireguard":{"implementation":"kernel","preserveClientEndpoint":true}}` | Set the options for the default gateway (server/client) templates. The default templates use a WireGuard implementation to connect the gateway of the clusters. These options are used to configure only the default templates and should not be considered if a custom template is used. | +| networking.gatewayTemplates | object | `{"container":{"gateway":{"extraArgs":[],"image":{"name":"ghcr.io/castai/images/liqo/gateway","version":""},"resources":{"limits":{},"requests":{}}},"geneve":{"image":{"name":"ghcr.io/castai/images/liqo/gateway/geneve","version":""},"resources":{"limits":{},"requests":{}}},"wireguard":{"image":{"name":"ghcr.io/castai/images/liqo/gateway/wireguard","version":""},"resources":{"limits":{},"requests":{}}}},"nftablesMonitor":true,"ping":{"interval":"2s","lossThreshold":5,"updateStatusInterval":"10s"},"pod":{"affinity":{},"annotations":{},"nodeSelector":{},"priorityClassName":"","tolerations":[]},"routeMonitor":true,"server":{"service":{"allocateLoadBalancerNodePorts":"","annotations":{}}},"wireguard":{"implementation":"kernel","preserveClientEndpoint":true}}` | Set the options for the default gateway (server/client) templates. The default templates use a WireGuard implementation to connect the gateway of the clusters. These options are used to configure only the default templates and should not be considered if a custom template is used. | | networking.gatewayTemplates.container.gateway.extraArgs | list | `[]` | Extra arguments for the gateway container. | -| networking.gatewayTemplates.container.gateway.image.name | string | `"ghcr.io/liqotech/gateway"` | Image repository for the gateway container. | +| networking.gatewayTemplates.container.gateway.image.name | string | `"ghcr.io/castai/images/liqo/gateway"` | Image repository for the gateway container. | | networking.gatewayTemplates.container.gateway.image.version | string | `""` | Custom version for the gateway image. If not specified, the global tag is used. | | networking.gatewayTemplates.container.gateway.resources | object | `{"limits":{},"requests":{}}` | Resource requests and limits (https://kubernetes.io/docs/user-guide/compute-resources/) for the gateway container. | -| networking.gatewayTemplates.container.geneve.image.name | string | `"ghcr.io/liqotech/gateway/geneve"` | Image repository for the geneve container. | +| networking.gatewayTemplates.container.geneve.image.name | string | `"ghcr.io/castai/images/liqo/gateway/geneve"` | Image repository for the geneve container. | | networking.gatewayTemplates.container.geneve.image.version | string | `""` | Custom version for the geneve image. If not specified, the global tag is used. | | networking.gatewayTemplates.container.geneve.resources | object | `{"limits":{},"requests":{}}` | Resource requests and limits (https://kubernetes.io/docs/user-guide/compute-resources/) for the geneve container. | -| networking.gatewayTemplates.container.wireguard.image.name | string | `"ghcr.io/liqotech/gateway/wireguard"` | Image repository for the wireguard container. | +| networking.gatewayTemplates.container.wireguard.image.name | string | `"ghcr.io/castai/images/liqo/gateway/wireguard"` | Image repository for the wireguard container. | | networking.gatewayTemplates.container.wireguard.image.version | string | `""` | Custom version for the wireguard image. If not specified, the global tag is used. | | networking.gatewayTemplates.container.wireguard.resources | object | `{"limits":{},"requests":{}}` | Resource requests and limits (https://kubernetes.io/docs/user-guide/compute-resources/) for the wireguard container. | | networking.gatewayTemplates.nftablesMonitor | bool | `true` | Enable/Disable the nftables monitor for the gateway pods. It means that the gateway pods will monitor the nftables rules and will restore them in case of changes. | @@ -143,7 +144,6 @@ | networking.gatewayTemplates.pod.nodeSelector | object | `{}` | NodeSelector for the gateway pods. | | networking.gatewayTemplates.pod.priorityClassName | string | `""` | PriorityClassName (https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority) for the gateway pods. | | networking.gatewayTemplates.pod.tolerations | list | `[]` | Additional tolerations for gateway pods | -| networking.gatewayTemplates.replicas | int | `1` | Set the number of replicas for the gateway deployments | | networking.gatewayTemplates.routeMonitor | bool | `true` | Enable/Disable the route monitor for the gateway pods. It means that the gateway pods will monitor the routing rules and will restore them in case of changes. | | networking.gatewayTemplates.server | object | `{"service":{"allocateLoadBalancerNodePorts":"","annotations":{}}}` | Set the options to configure the gateway server | | networking.gatewayTemplates.server.service | object | `{"allocateLoadBalancerNodePorts":"","annotations":{}}` | Set the options to configure the server service | @@ -170,8 +170,12 @@ | offloading.reflection.ingress.ingressClasses | list | `[]` | List of ingress classes that will be shown to remote clusters. If empty, ingress class will be reflected as-is. Example: ingressClasses: - name: nginx default: true - name: traefik | | offloading.reflection.ingress.type | string | `"DenyList"` | The type of reflection used for the ingresses reflector. Ammitted values: "DenyList", "AllowList". | | offloading.reflection.ingress.workers | int | `3` | The number of workers used for the ingresses reflector. Set 0 to disable the reflection of ingresses. | +| offloading.reflection.localstoragepvc.type | string | `"DenyList"` | The type of reflection used for the pvc mapped on local storage reflector. Ammitted values: "DenyList", "AllowList". | +| offloading.reflection.localstoragepvc.workers | int | `1` | The number of workers used for the localstoragepvcs reflector. Set 0 to disable the reflection of localstoragepvcs. | | offloading.reflection.persistentvolumeclaim.workers | int | `3` | The number of workers used for the persistentvolumeclaims reflector. Set 0 to disable the reflection of persistentvolumeclaims. | | offloading.reflection.pod.workers | int | `10` | The number of workers used for the pods reflector. Set 0 to disable the reflection of pods. | +| offloading.reflection.resourceclaim.workers | int | `3` | The number of workers used for the DRA ResourceClaim reflector. Set 0 to disable the reflection of ResourceClaims. Reflection is also automatically skipped if the resource.k8s.io/v1 API is not available on either cluster. | +| offloading.reflection.resourceslice.workers | int | `1` | The number of workers used for the DRA ResourceSlice reflector (leader-only). Set 0 to disable the reflection of ResourceSlices. Reflection is also automatically skipped if the resource.k8s.io/v1 API is not available on either cluster. | | offloading.reflection.secret.type | string | `"DenyList"` | The type of reflection used for the secrets reflector. Ammitted values: "DenyList", "AllowList". | | offloading.reflection.secret.workers | int | `3` | The number of workers used for the secrets reflector. Set 0 to disable the reflection of secrets. | | offloading.reflection.service.loadBalancerClasses | list | `[]` | List of load balancer classes that will be shown to remote clusters. If empty, load balancer classes will be reflected as-is. Example: loadBalancerClasses: - name: public default: true - name: internal | @@ -184,15 +188,15 @@ | offloading.runtimeClass.handler | string | `"liqo"` | Handler for the runtime class. | | offloading.runtimeClass.labels | object | `{}` | Labels for the runtime class. | | offloading.runtimeClass.name | string | `"liqo"` | Name of the runtime class to use for offloading. | -| offloading.runtimeClass.nodeSelector | object | `{"enabled":true,"labels":{"liqo.io/type":"virtual-node"}}` | Node selector for the runtime class. | -| offloading.runtimeClass.nodeSelector.labels | object | `{"liqo.io/type":"virtual-node"}` | Labels for the node selector. | -| offloading.runtimeClass.tolerations | object | `{"enabled":true,"tolerations":[{"effect":"NoExecute","key":"virtual-node.liqo.io/not-allowed","operator":"Exists"}]}` | Tolerations for the runtime class. | -| offloading.runtimeClass.tolerations.tolerations | list | `[{"effect":"NoExecute","key":"virtual-node.liqo.io/not-allowed","operator":"Exists"}]` | Tolerations for the tolerations. | +| offloading.runtimeClass.nodeSelector | object | `{"enabled":true,"labels":{"omni.cast.ai/type":"virtual-node"}}` | Node selector for the runtime class. | +| offloading.runtimeClass.nodeSelector.labels | object | `{"omni.cast.ai/type":"virtual-node"}` | Labels for the node selector. | +| offloading.runtimeClass.tolerations | object | `{"enabled":true,"tolerations":[{"effect":"NoExecute","key":"virtual-node.omni.cast.ai/not-allowed","operator":"Exists"}]}` | Tolerations for the runtime class. | +| offloading.runtimeClass.tolerations.tolerations | list | `[{"effect":"NoExecute","key":"virtual-node.omni.cast.ai/not-allowed","operator":"Exists"}]` | Tolerations for the tolerations. | | openshiftConfig.enabled | bool | `false` | Enable/Disable the OpenShift support, enabling Openshift-specific resources, and setting the pod security contexts in a way that is compatible with Openshift. | | openshiftConfig.virtualKubeletSCCs | list | `["anyuid","privileged"]` | Security context configurations granted to the virtual kubelet in the local cluster. The configuration of one or more SCCs for the virtual kubelet is not strictly required, and privileges can be reduced in production environments. Still, the default configuration (i.e., anyuid) is suggested to prevent problems (i.e., the virtual kubelet fails to add the appropriate labels) when attempting to offload pods not managed by higher-level abstractions (e.g., Deployments), and not associated with a properly privileged service account. Indeed, "anyuid" is the SCC automatically associated with pods created by cluster administrators. Any pod granted a more privileged SCC and not linked to an adequately privileged service account will fail to be offloaded. | | proxy.config.listeningPort | int | `8118` | Port used by the proxy pod. | | proxy.enabled | bool | `true` | Enable/Disable the proxy pod. This pod is mandatory to allow in-band peering and to connect to the consumer k8s api server from a remotly offloaded pod. | -| proxy.image.name | string | `"ghcr.io/liqotech/proxy"` | Image repository for the proxy pod. | +| proxy.image.name | string | `"ghcr.io/castai/images/liqo/proxy"` | Image repository for the proxy pod. | | proxy.image.version | string | `""` | Custom version for the proxy image. If not specified, the global tag is used. | | proxy.pod.annotations | object | `{}` | Annotations for the proxy pod. | | proxy.pod.extraArgs | list | `[]` | Extra arguments for the proxy pod. | @@ -211,24 +215,26 @@ | tag | string | `""` | Images' tag to select a development version of liqo instead of a release | | telemetry.config.schedule | string | `""` | Set the schedule of the telemetry collector CronJob. Consider setting this value on ArgoCD deployments to avoid randomization. | | telemetry.enabled | bool | `true` | Enable/Disable the telemetry collector. | -| telemetry.image.name | string | `"ghcr.io/liqotech/telemetry"` | Image repository for the telemetry pod. | +| telemetry.image.name | string | `"ghcr.io/castai/images/liqo/telemetry"` | Image repository for the telemetry pod. | | telemetry.image.version | string | `""` | Custom version for the telemetry image. If not specified, the global tag is used. | | telemetry.pod.annotations | object | `{}` | Annotations for the telemetry pod. | | telemetry.pod.extraArgs | list | `[]` | Extra arguments for the telemetry pod. | | telemetry.pod.labels | object | `{}` | Labels for the telemetry pod. | | telemetry.pod.resources | object | `{"limits":{},"requests":{}}` | Resource requests and limits (https://kubernetes.io/docs/user-guide/compute-resources/) for the telemetry pod. | -| uninstaller.image.name | string | `"ghcr.io/liqotech/uninstaller"` | Image repository for the uninstaller pod. | +| uninstaller.image.name | string | `"ghcr.io/castai/images/liqo/uninstaller"` | Image repository for the uninstaller pod. | | uninstaller.image.version | string | `""` | Custom version for the uninstaller image. If not specified, the global tag is used. | | uninstaller.pod.annotations | object | `{}` | Annotations for the uninstaller pod. | | uninstaller.pod.extraArgs | list | `[]` | Extra arguments for the uninstaller pod. | | uninstaller.pod.labels | object | `{}` | Labels for the uninstaller pod. | | uninstaller.pod.resources | object | `{"limits":{},"requests":{}}` | Resource requests and limits (https://kubernetes.io/docs/user-guide/compute-resources/) for the uninstaller pod. | +| virtualKubelet.extra.affinity | object | `{}` | | | virtualKubelet.extra.annotations | object | `{}` | Annotations for the virtual kubelet pod. | | virtualKubelet.extra.args | list | `[]` | Extra arguments virtual kubelet pod. | | virtualKubelet.extra.labels | object | `{}` | Labels for the virtual kubelet pod. | +| virtualKubelet.extra.nodeSelector | object | `{}` | | | virtualKubelet.extra.resources | object | `{"limits":{},"requests":{}}` | Resource requests and limits (https://kubernetes.io/docs/user-guide/compute-resources/) for the virtual kubelet pod. | | virtualKubelet.extra.tolerations | list | `[]` | Tolerations for the virtual kubelet pod. | -| virtualKubelet.image.name | string | `"ghcr.io/liqotech/virtual-kubelet"` | Image repository for the virtual kubelet pod. | +| virtualKubelet.image.name | string | `"ghcr.io/castai/images/liqo/virtual-kubelet"` | Image repository for the virtual kubelet pod. | | virtualKubelet.image.version | string | `""` | Custom version for the virtual kubelet image. If not specified, the global tag is used. | | virtualKubelet.metrics.podMonitor.interval | string | `""` | Setup pod monitor requests interval. If empty, Prometheus uses the global scrape interval (https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint). | | virtualKubelet.metrics.podMonitor.labels | object | `{}` | Labels for the virtualkubelet podmonitor. | @@ -238,7 +244,7 @@ | virtualKubelet.virtualNode.extra.annotations | object | `{}` | Extra annotations for the virtual node. | | virtualKubelet.virtualNode.extra.labels | object | `{}` | Extra labels for the virtual node. | | webhook.failurePolicy | string | `"Fail"` | Webhook failure policy, either Ignore or Fail. | -| webhook.image.name | string | `"ghcr.io/liqotech/webhook"` | Image repository for the webhook pod. | +| webhook.image.name | string | `"ghcr.io/castai/images/liqo/webhook"` | Image repository for the webhook pod. | | webhook.image.version | string | `""` | Custom version for the webhook image. If not specified, the global tag is used. | | webhook.metrics.service | object | `{"annotations":{},"labels":{}}` | Service used to expose metrics. | | webhook.metrics.service.annotations | object | `{}` | Annotations for the metrics service. | diff --git a/deployments/liqo/charts/liqo-crds/crds/networking.liqo.io_firewallconfigurations.yaml b/deployments/liqo/charts/liqo-crds/crds/networking.liqo.io_firewallconfigurations.yaml index ca44f1a1c9..a3424f5884 100644 --- a/deployments/liqo/charts/liqo-crds/crds/networking.liqo.io_firewallconfigurations.yaml +++ b/deployments/liqo/charts/liqo-crds/crds/networking.liqo.io_firewallconfigurations.yaml @@ -108,6 +108,7 @@ spec: - drop - reject - notrack + - setmetamark type: string counter: default: true @@ -137,6 +138,13 @@ spec: description: Value is the name of the device to be matched. type: string + wildcard: + default: false + description: |- + Wildcard enables prefix matching on the device name. + When true, the match compares only the prefix bytes of Value against the interface name, + allowing a single rule to match all interfaces sharing a common prefix (e.g. "liqo."). + type: boolean required: - position - value @@ -247,6 +255,13 @@ spec: description: Value is the name of the device to be matched. type: string + wildcard: + default: false + description: |- + Wildcard enables prefix matching on the device name. + When true, the match compares only the prefix bytes of Value against the interface name, + allowing a single rule to match all interfaces sharing a common prefix (e.g. "liqo."). + type: boolean required: - position - value diff --git a/deployments/liqo/charts/liqo-crds/crds/networking.liqo.io_internalfabrics.yaml b/deployments/liqo/charts/liqo-crds/crds/networking.liqo.io_internalfabrics.yaml index 8381edbf33..383d0a34e7 100644 --- a/deployments/liqo/charts/liqo-crds/crds/networking.liqo.io_internalfabrics.yaml +++ b/deployments/liqo/charts/liqo-crds/crds/networking.liqo.io_internalfabrics.yaml @@ -103,7 +103,17 @@ spec: - gatewayIP - interface type: object + status: + description: InternalFabricStatus defines the observed state of InternalFabric. + properties: + ecmpMark: + description: |- + ECMPMark is the policy-routing fwmark assigned to this gateway for ECMP consistency. + It is allocated by the controller and persisted here so it remains stable across reconnects. + type: integer + type: object type: object served: true storage: true - subresources: {} + subresources: + status: {} diff --git a/deployments/liqo/charts/liqo-crds/crds/offloading.liqo.io_vkoptionstemplates.yaml b/deployments/liqo/charts/liqo-crds/crds/offloading.liqo.io_vkoptionstemplates.yaml index a1ee0adb4b..fc09952b62 100644 --- a/deployments/liqo/charts/liqo-crds/crds/offloading.liqo.io_vkoptionstemplates.yaml +++ b/deployments/liqo/charts/liqo-crds/crds/offloading.liqo.io_vkoptionstemplates.yaml @@ -49,6 +49,918 @@ spec: spec: description: VkOptionsTemplateSpec defines the desired state of VkOptionsTemplate. properties: + affinity: + description: Affinity is a group of affinity scheduling rules. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the + pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the + corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object annotationsNotReflected: items: type: string @@ -105,6 +1017,10 @@ spec: additionalProperties: type: string type: object + nodeSelector: + additionalProperties: + type: string + type: object pullPolicy: description: PullPolicy describes a policy for if/when to pull a container image diff --git a/deployments/liqo/charts/liqo-crds/values.yaml b/deployments/liqo/charts/liqo-crds/values.yaml index bb42933f6b..69915682d3 100644 --- a/deployments/liqo/charts/liqo-crds/values.yaml +++ b/deployments/liqo/charts/liqo-crds/values.yaml @@ -9,7 +9,7 @@ crdUpgrade: # image is the CRD upgrade container image # NOTE: This image only supports linux/amd64 and linux/arm64 platforms (arm/v7 is not supported). image: - name: "ghcr.io/liqotech/liqo-crd-upgrade" + name: "ghcr.io/castai/images/liqo/liqo-crd-upgrade" # version: REQUIRED when crdUpgrade.enabled is true. The image tag/version to use. # Set using: --set liqo-crds.crdUpgrade.image.version= version: "" diff --git a/deployments/liqo/files/liqo-controller-manager-ClusterRole.yaml b/deployments/liqo/files/liqo-controller-manager-ClusterRole.yaml index 72e5117887..6f96f0d339 100644 --- a/deployments/liqo/files/liqo-controller-manager-ClusterRole.yaml +++ b/deployments/liqo/files/liqo-controller-manager-ClusterRole.yaml @@ -220,6 +220,7 @@ rules: - networking.liqo.io resources: - configurations/finalizers + - firewallconfigurations/finalizers - gatewayclients/finalizers - gatewayservers/finalizers - genevetunnels/finalizers diff --git a/deployments/liqo/files/liqo-gateway-ClusterRole.yaml b/deployments/liqo/files/liqo-gateway-ClusterRole.yaml index 4431f5e693..b58b5b19f2 100644 --- a/deployments/liqo/files/liqo-gateway-ClusterRole.yaml +++ b/deployments/liqo/files/liqo-gateway-ClusterRole.yaml @@ -11,25 +11,6 @@ rules: - patch - update - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - delete - - get - - update - apiGroups: - networking.liqo.io resources: diff --git a/deployments/liqo/files/liqo-virtual-kubelet-local-ClusterRole.yaml b/deployments/liqo/files/liqo-virtual-kubelet-local-ClusterRole.yaml index 66552ee936..640dfbeb51 100644 --- a/deployments/liqo/files/liqo-virtual-kubelet-local-ClusterRole.yaml +++ b/deployments/liqo/files/liqo-virtual-kubelet-local-ClusterRole.yaml @@ -124,6 +124,27 @@ rules: - get - list - watch +- apiGroups: + - resource.k8s.io + resources: + - deviceclasses + - resourceclaims + verbs: + - get + - list + - watch +- apiGroups: + - resource.k8s.io + resources: + - resourceslices + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - storage.k8s.io resources: diff --git a/deployments/liqo/files/liqo-virtual-kubelet-remote-ClusterRole.yaml b/deployments/liqo/files/liqo-virtual-kubelet-remote-ClusterRole.yaml index 2ffe79b206..9064e25555 100644 --- a/deployments/liqo/files/liqo-virtual-kubelet-remote-ClusterRole.yaml +++ b/deployments/liqo/files/liqo-virtual-kubelet-remote-ClusterRole.yaml @@ -70,3 +70,15 @@ rules: - patch - update - watch +- apiGroups: + - resource.k8s.io + resources: + - resourceclaims + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/deployments/liqo/files/liqo-virtual-kubelet-remote-clusterwide-ClusterRole.yaml b/deployments/liqo/files/liqo-virtual-kubelet-remote-clusterwide-ClusterRole.yaml index fa4f38942d..7ae6c34582 100644 --- a/deployments/liqo/files/liqo-virtual-kubelet-remote-clusterwide-ClusterRole.yaml +++ b/deployments/liqo/files/liqo-virtual-kubelet-remote-clusterwide-ClusterRole.yaml @@ -1,4 +1,24 @@ rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - metrics.liqo.io resources: @@ -7,3 +27,20 @@ rules: - get - list - watch +- apiGroups: + - resource.k8s.io + resources: + - deviceclasses + verbs: + - create + - get + - list + - watch +- apiGroups: + - resource.k8s.io + resources: + - resourceslices + verbs: + - get + - list + - watch diff --git a/deployments/liqo/templates/_helpers.tpl b/deployments/liqo/templates/_helpers.tpl index d560312c42..efdff063ba 100644 --- a/deployments/liqo/templates/_helpers.tpl +++ b/deployments/liqo/templates/_helpers.tpl @@ -95,9 +95,7 @@ Common Labels for Gateway Templates {{ include "liqo.selectorTemplate" . }} helm.sh/chart: {{ quote (include "liqo.chart" .) }} app.kubernetes.io/version: {{ quote (include "liqo.version" .) }} -{{- if and .isGwTemplate (eq (int .Values.networking.gatewayTemplates.replicas) 1) }} -networking.liqo.io/active: "true" -{{- end }} +liqo.io/remote-cluster-id: "{{"{{ .ClusterID }}"}}" {{- end }} {{/* @@ -109,9 +107,6 @@ app.kubernetes.io/managed-by: {{ quote .Release.Service }} networking.liqo.io/component: "gateway" networking.liqo.io/gateway-name: "{{"{{ .Name }}"}}" networking.liqo.io/gateway-namespace: "{{"{{ .Namespace }}"}}" -{{- if .isService }} -networking.liqo.io/active: "true" -{{- end }} {{- end }} {{/* diff --git a/deployments/liqo/templates/liqo-fabric-daemonset.yaml b/deployments/liqo/templates/liqo-fabric-daemonset.yaml index 1e0536a8b1..cf8fd304a3 100644 --- a/deployments/liqo/templates/liqo-fabric-daemonset.yaml +++ b/deployments/liqo/templates/liqo-fabric-daemonset.yaml @@ -69,6 +69,7 @@ spec: {{- end }} - --enable-nft-monitor={{ .Values.networking.fabric.config.nftablesMonitor }} - --enable-route-monitor={{ .Values.networking.fabric.config.routeMonitor }} + - --enable-multipath-hash-policy={{ .Values.networking.fabric.config.multipathHashPolicy }} {{- if .Values.common.globalAnnotations }} {{- $d := dict "commandName" "--global-annotations" "dictionary" .Values.common.globalAnnotations -}} {{- include "liqo.concatenateMap" $d | nindent 10 }} diff --git a/deployments/liqo/templates/liqo-vk-options-template.yaml b/deployments/liqo/templates/liqo-vk-options-template.yaml index 19fb33ac4b..0f01249ac8 100644 --- a/deployments/liqo/templates/liqo-vk-options-template.yaml +++ b/deployments/liqo/templates/liqo-vk-options-template.yaml @@ -46,7 +46,7 @@ spec: {{- end }} {{- include "liqo.imagePullSecrets" . | nindent 2 }} reflectorsConfig: - pod: + pod: workers: {{ .Values.offloading.reflection.pod.workers }} service: workers: {{ .Values.offloading.reflection.service.workers }} @@ -66,9 +66,16 @@ spec: workers: {{ .Values.offloading.reflection.serviceaccount.workers }} persistentvolumeclaim: workers: {{ .Values.offloading.reflection.persistentvolumeclaim.workers }} + localstoragepvc: + workers: {{ .Values.offloading.reflection.localstoragepvc.workers }} + type: {{ .Values.offloading.reflection.localstoragepvc.type }} event: workers: {{ .Values.offloading.reflection.event.workers }} type: {{ .Values.offloading.reflection.event.type }} + resourceslice: + workers: {{ .Values.offloading.reflection.resourceslice.workers }} + resourceclaim: + workers: {{ .Values.offloading.reflection.resourceclaim.workers }} {{- if .Values.virtualKubelet.extra.resources }} resources: {{- toYaml .Values.virtualKubelet.extra.resources | nindent 4 }} @@ -97,3 +104,11 @@ spec: tolerations: {{- toYaml .Values.virtualKubelet.extra.tolerations | nindent 4 }} {{- end }} + {{- if .Values.virtualKubelet.extra.affinity }} + affinity: + {{- toYaml .Values.virtualKubelet.extra.affinity | nindent 4 }} + {{- end }} + {{- if .Values.virtualKubelet.extra.nodeSelector }} + nodeSelector: + {{- toYaml .Values.virtualKubelet.extra.nodeSelector | nindent 4 }} + {{- end }} diff --git a/deployments/liqo/templates/liqo-wireguard-gateway-client-template.yaml b/deployments/liqo/templates/liqo-wireguard-gateway-client-template.yaml index 2d7015a445..8547bc6abe 100644 --- a/deployments/liqo/templates/liqo-wireguard-gateway-client-template.yaml +++ b/deployments/liqo/templates/liqo-wireguard-gateway-client-template.yaml @@ -25,7 +25,6 @@ spec: metadata: {{- include "liqo.metadataTemplate" $templateConfig | nindent 10 }} spec: - replicas: {{ .Values.networking.gatewayTemplates.replicas }} strategy: type: Recreate selector: @@ -70,8 +69,6 @@ spec: - --pod-name={{"$(POD_NAME)"}} - --gateway-uid={{"{{ .GatewayUID }}"}} - --mode=client - - --container-name=gateway - - --concurrent-containers-names=wireguard,geneve {{- if .Values.common.globalAnnotations }} {{- $d := dict "commandName" "--global-annotations" "dictionary" .Values.common.globalAnnotations -}} {{- include "liqo.concatenateMap" $d | nindent 16 }} @@ -88,23 +85,15 @@ spec: - --ping-loss-threshold={{ .Values.networking.gatewayTemplates.ping.lossThreshold }} - --ping-interval={{ .Values.networking.gatewayTemplates.ping.interval }} - --ping-update-status-interval={{ .Values.networking.gatewayTemplates.ping.updateStatusInterval }} - {{- if gt (int .Values.networking.gatewayTemplates.replicas) 1 }} - - --leader-election=true - {{- else }} - - --leader-election=false - {{- end }} {{- if not .Values.requirements.kernel.enabled }} - --disable-kernel-version-check {{- end }} - --enable-nft-monitor={{ .Values.networking.gatewayTemplates.nftablesMonitor }} - --enable-route-monitor={{ .Values.networking.gatewayTemplates.routeMonitor }} - - --enable-multipath-hash-policy={{ printf "{{ if gt (len .Spec.Endpoint.Ports) 1 }}true{{ else }}false{{ end }}" }} + - --enable-multipath-hash-policy=true {{- if .Values.networking.gatewayTemplates.container.gateway.extraArgs }} {{- toYaml .Values.networking.gatewayTemplates.container.gateway.extraArgs | nindent 16 }} {{- end }} - volumeMounts: - - name: ipc - mountPath: /ipc ports: {{- if .Values.metrics.enabled }} - containerPort: 8082 @@ -146,7 +135,6 @@ spec: - --remote-cluster-id={{"{{ .ClusterID }}"}} - --gateway-uid={{"{{ .GatewayUID }}"}} - --mode=client - - --container-name=wireguard - --mtu={{"{{ .Spec.MTU }}"}} - --endpoint-address={{"{{ index .Spec.Endpoint.Addresses 0 }}"}} - --endpoint-ports={{ printf "{{ if .Spec.Endpoint.Ports }}{{ range $i, $p := .Spec.Endpoint.Ports }}{{ if $i }},{{ end }}{{ $p }}{{ end }}{{ else }}{{ .Spec.Endpoint.Port }}{{ end }}" }} @@ -154,11 +142,6 @@ spec: - --metrics-address=:8084 {{- end }} - --health-probe-bind-address=:8085 - {{- if gt (int .Values.networking.gatewayTemplates.replicas) 1 }} - - --leader-election=true - {{- else }} - - --leader-election=false - {{- end }} - --implementation={{ .Values.networking.gatewayTemplates.wireguard.implementation }} ports: {{- if .Values.metrics.enabled }} @@ -183,8 +166,6 @@ spec: privileged: true {{ end }} volumeMounts: - - name: ipc - mountPath: /ipc - name: wireguard-config mountPath: /etc/wireguard/keys - name: geneve @@ -202,7 +183,6 @@ spec: - --pod-name={{"$(POD_NAME)"}} - --gateway-uid={{"{{ .GatewayUID }}"}} - --mode=server - - --container-name=geneve - --geneve-port={{ .Values.networking.genevePort }} - --ping-enabled=true - --ping-port={{ .Values.networking.fabric.config.ping.port }} @@ -210,14 +190,6 @@ spec: - --metrics-address=:8086 {{- end }} - --health-probe-bind-address=:8087 - {{- if gt (int .Values.networking.gatewayTemplates.replicas) 1 }} - - --leader-election=true - {{- else }} - - --leader-election=false - {{- end }} - volumeMounts: - - name: ipc - mountPath: /ipc ports: {{- if .Values.metrics.enabled }} - containerPort: 8086 @@ -252,6 +224,4 @@ spec: - name: wireguard-config secret: secretName: "{{"{{ .SecretName }}"}}" - - name: ipc - emptyDir: {} {{- end }} diff --git a/deployments/liqo/templates/liqo-wireguard-gateway-server-template.yaml b/deployments/liqo/templates/liqo-wireguard-gateway-server-template.yaml index aaa1f97db1..ebd839b319 100644 --- a/deployments/liqo/templates/liqo-wireguard-gateway-server-template.yaml +++ b/deployments/liqo/templates/liqo-wireguard-gateway-server-template.yaml @@ -45,7 +45,6 @@ spec: metadata: {{- include "liqo.metadataTemplate" $templateConfig | nindent 10 }} spec: - replicas: {{ .Values.networking.gatewayTemplates.replicas }} strategy: type: Recreate selector: @@ -90,8 +89,6 @@ spec: - --pod-name={{"$(POD_NAME)"}} - --gateway-uid={{"{{ .GatewayUID }}"}} - --mode=server - - --container-name=gateway - - --concurrent-containers-names=wireguard,geneve {{- if .Values.common.globalAnnotations }} {{- $d := dict "commandName" "--global-annotations" "dictionary" .Values.common.globalAnnotations -}} {{- include "liqo.concatenateMap" $d | nindent 16 }} @@ -108,23 +105,15 @@ spec: - --ping-loss-threshold={{ .Values.networking.gatewayTemplates.ping.lossThreshold }} - --ping-interval={{ .Values.networking.gatewayTemplates.ping.interval }} - --ping-update-status-interval={{ .Values.networking.gatewayTemplates.ping.updateStatusInterval }} - {{- if gt (int .Values.networking.gatewayTemplates.replicas) 1 }} - - --leader-election=true - {{- else }} - - --leader-election=false - {{- end }} {{- if not .Values.requirements.kernel.enabled }} - --disable-kernel-version-check {{- end }} - --enable-nft-monitor={{ .Values.networking.gatewayTemplates.nftablesMonitor }} - --enable-route-monitor={{ .Values.networking.gatewayTemplates.routeMonitor }} - - --enable-multipath-hash-policy={{ printf "{{ if gt (len .Spec.Endpoint.Ports) 1 }}true{{ else }}false{{ end }}" }} + - --enable-multipath-hash-policy=true {{- if .Values.networking.gatewayTemplates.container.gateway.extraArgs }} {{- toYaml .Values.networking.gatewayTemplates.container.gateway.extraArgs | nindent 16 }} {{- end }} - volumeMounts: - - name: ipc - mountPath: /ipc ports: {{- if .Values.metrics.enabled }} - containerPort: 8082 @@ -166,18 +155,12 @@ spec: - --remote-cluster-id={{"{{ .ClusterID }}"}} - --gateway-uid={{"{{ .GatewayUID }}"}} - --mode=server - - --container-name=wireguard - --mtu={{"{{ .Spec.MTU }}"}} - --listen-ports={{ printf "{{ if .Spec.Endpoint.Ports }}{{ range $i, $p := .Spec.Endpoint.Ports }}{{ if $i }},{{ end }}{{ $p }}{{ end }}{{ else }}{{ .Spec.Endpoint.Port }}{{ end }}" }} {{- if .Values.metrics.enabled }} - --metrics-address=:8084 {{- end }} - --health-probe-bind-address=:8085 - {{- if gt (int .Values.networking.gatewayTemplates.replicas) 1 }} - - --leader-election=true - {{- else }} - - --leader-election=false - {{- end }} - --implementation={{ .Values.networking.gatewayTemplates.wireguard.implementation }} - --preserve-client-endpoint={{ .Values.networking.gatewayTemplates.wireguard.preserveClientEndpoint }} ports: @@ -203,8 +186,6 @@ spec: privileged: true {{ end }} volumeMounts: - - name: ipc - mountPath: /ipc - name: wireguard-config mountPath: /etc/wireguard/keys - name: geneve @@ -222,7 +203,6 @@ spec: - --pod-name={{"$(POD_NAME)"}} - --gateway-uid={{"{{ .GatewayUID }}"}} - --mode=server - - --container-name=geneve - --geneve-port={{ .Values.networking.genevePort }} - --ping-enabled=true - --ping-port={{ .Values.networking.fabric.config.ping.port }} @@ -230,14 +210,6 @@ spec: - --metrics-address=:8086 {{- end }} - --health-probe-bind-address=:8087 - {{- if gt (int .Values.networking.gatewayTemplates.replicas) 1 }} - - --leader-election=true - {{- else }} - - --leader-election=false - {{- end }} - volumeMounts: - - name: ipc - mountPath: /ipc ports: {{- if .Values.metrics.enabled }} - containerPort: 8086 @@ -272,6 +244,4 @@ spec: - name: wireguard-config secret: secretName: "{{"{{ .SecretName }}"}}" - - name: ipc - emptyDir: {} {{- end }} diff --git a/deployments/liqo/values.yaml b/deployments/liqo/values.yaml index 31bbe845ff..f65b6def20 100644 --- a/deployments/liqo/values.yaml +++ b/deployments/liqo/values.yaml @@ -29,7 +29,7 @@ liqo-crds: image: # -- Image repository for the CRD upgrade job. # NOTE: This image only supports linux/amd64 and linux/arm64 platforms (arm/v7 is not supported). - name: "ghcr.io/liqotech/liqo-crd-upgrade" + name: "ghcr.io/castai/images/liqo/liqo-crd-upgrade" # -- Image version for the CRD upgrade job. Required when crdUpgrade.enabled is true. version: "" # -- Image pull policy for the CRD upgrade job. @@ -105,8 +105,6 @@ networking: # -- Preserve the existing peer endpoint when configuring the server-side WireGuard device. # This preserves the discovered endpoint across PublicKey reconciliations. preserveClientEndpoint: true - # -- Set the number of replicas for the gateway deployments - replicas: 1 # -- Set the options to configure the gateway ping used to check connection ping: # -- Set the number of consecutive pings that must fail to consider the connection as lost @@ -138,7 +136,7 @@ networking: gateway: image: # -- Image repository for the gateway container. - name: "ghcr.io/liqotech/gateway" + name: "ghcr.io/castai/images/liqo/gateway" # -- Custom version for the gateway image. If not specified, the global tag is used. version: "" # -- Resource requests and limits (https://kubernetes.io/docs/user-guide/compute-resources/) for the gateway container. @@ -150,7 +148,7 @@ networking: wireguard: image: # -- Image repository for the wireguard container. - name: "ghcr.io/liqotech/gateway/wireguard" + name: "ghcr.io/castai/images/liqo/gateway/wireguard" # -- Custom version for the wireguard image. If not specified, the global tag is used. version: "" # -- Resource requests and limits (https://kubernetes.io/docs/user-guide/compute-resources/) for the wireguard container. @@ -160,7 +158,7 @@ networking: geneve: image: # -- Image repository for the geneve container. - name: "ghcr.io/liqotech/gateway/geneve" + name: "ghcr.io/castai/images/liqo/gateway/geneve" # -- Custom version for the geneve image. If not specified, the global tag is used. version: "" # -- Resource requests and limits (https://kubernetes.io/docs/user-guide/compute-resources/) for the geneve container. @@ -189,7 +187,7 @@ networking: priorityClassName: "" image: # -- Image repository for the fabric pod. - name: "ghcr.io/liqotech/fabric" + name: "ghcr.io/castai/images/liqo/fabric" # -- Custom version for the fabric image. If not specified, the global tag is used. version: "" # -- NodeSelector for the fabric pod. @@ -233,6 +231,9 @@ networking: # -- Enable/Disable the route monitor for the fabric pod. # It means that the fabric pod will monitor the routing rules and will restore them in case of changes. routeMonitor: true + # -- Enable 5-tuple ECMP hashing (fib_multipath_hash_policy=1) in the fabric pod. + # This improves load balancing when multiple gateway replicas provide ECMP next-hops. + multipathHashPolicy: true # -- Set the port where the fabric pod will expose the health probe. # To disable the health probe, set the port to 0. healthProbeBindAddressPort: "8081" @@ -327,13 +328,13 @@ offloading: enabled: true # -- Labels for the node selector. labels: - liqo.io/type: virtual-node + omni.cast.ai/type: virtual-node # -- Tolerations for the runtime class. tolerations: enabled: true # -- Tolerations for the tolerations. tolerations: - - key: virtual-node.liqo.io/not-allowed + - key: virtual-node.omni.cast.ai/not-allowed operator: Exists effect: NoExecute reflection: @@ -401,11 +402,26 @@ offloading: persistentvolumeclaim: # -- The number of workers used for the persistentvolumeclaims reflector. Set 0 to disable the reflection of persistentvolumeclaims. workers: 3 + localstoragepvc: + # -- The number of workers used for the localstoragepvcs reflector. Set 0 to disable the reflection of localstoragepvcs. + workers: 1 + # -- The type of reflection used for the pvc mapped on local storage reflector. Ammitted values: "DenyList", "AllowList". + type: DenyList event: # -- The number of workers used for the events reflector. Set 0 to disable the reflection of events. workers: 3 # -- The type of reflection used for the events reflector. Ammitted values: "DenyList", "AllowList". type: DenyList + resourceslice: + # -- The number of workers used for the DRA ResourceSlice reflector (leader-only). + # Set 0 to disable the reflection of ResourceSlices. Reflection is also automatically + # skipped if the resource.k8s.io/v1 API is not available on either cluster. + workers: 1 + resourceclaim: + # -- The number of workers used for the DRA ResourceClaim reflector. Set 0 to disable + # the reflection of ResourceClaims. Reflection is also automatically skipped if the + # resource.k8s.io/v1 API is not available on either cluster. + workers: 3 storage: # -- Enable/Disable the liqo virtual storage class on the local cluster. You will be able to @@ -452,7 +468,7 @@ controllerManager: priorityClassName: "" image: # -- Image repository for the controller-manager pod. - name: "ghcr.io/liqotech/liqo-controller-manager" + name: "ghcr.io/castai/images/liqo/liqo-controller-manager" # -- Custom version for the controller-manager image. If not specified, the global tag is used. version: "" config: @@ -511,7 +527,7 @@ webhook: priorityClassName: "" image: # -- Image repository for the webhook pod. - name: "ghcr.io/liqotech/webhook" + name: "ghcr.io/castai/images/liqo/webhook" # -- Custom version for the webhook image. If not specified, the global tag is used. version: "" metrics: @@ -547,7 +563,7 @@ ipam: internal: image: # -- Image repository for the IPAM pod. - name: "ghcr.io/liqotech/ipam" + name: "ghcr.io/castai/images/liqo/ipam" # -- Custom version for the IPAM image. If not specified, the global tag is used. version: "" # -- The number of IPAM instances to run, which can be increased for active/passive high availability. @@ -634,7 +650,7 @@ crdReplicator: labels: {} image: # -- Image repository for the crdReplicator pod. - name: "ghcr.io/liqotech/crd-replicator" + name: "ghcr.io/castai/images/liqo/crd-replicator" # -- Custom version for the crdReplicator image. If not specified, the global tag is used. version: "" @@ -680,13 +696,13 @@ metricAgent: priorityClassName: "" image: # -- Image repository for the metricAgent pod. - name: "ghcr.io/liqotech/metric-agent" + name: "ghcr.io/castai/images/liqo/metric-agent" # -- Custom version for the metricAgent image. If not specified, the global tag is used. version: "" initContainer: image: # --Image repository for the init container of the metricAgent pod. - name: "ghcr.io/liqotech/cert-creator" + name: "ghcr.io/castai/images/liqo/cert-creator" # -- Custom version for the init container image of the metricAgent pod. If not specified, the global tag is used. version: "" @@ -706,7 +722,7 @@ telemetry: requests: {} image: # -- Image repository for the telemetry pod. - name: "ghcr.io/liqotech/telemetry" + name: "ghcr.io/castai/images/liqo/telemetry" # -- Custom version for the telemetry image. If not specified, the global tag is used. version: "" config: @@ -719,7 +735,7 @@ virtualKubelet: replicas: 1 image: # -- Image repository for the virtual kubelet pod. - name: "ghcr.io/liqotech/virtual-kubelet" + name: "ghcr.io/castai/images/liqo/virtual-kubelet" # -- Custom version for the virtual kubelet image. If not specified, the global tag is used. version: "" # Additional values that are added to virtual kubelet deployments and pods. @@ -736,6 +752,8 @@ virtualKubelet: requests: {} # -- Tolerations for the virtual kubelet pod. tolerations: [] + affinity: {} + nodeSelector: {} virtualNode: extra: # -- Extra annotations for the virtual node. @@ -769,7 +787,7 @@ uninstaller: requests: {} image: # -- Image repository for the uninstaller pod. - name: "ghcr.io/liqotech/uninstaller" + name: "ghcr.io/castai/images/liqo/uninstaller" # -- Custom version for the uninstaller image. If not specified, the global tag is used. version: "" @@ -795,7 +813,7 @@ proxy: priorityClassName: "" image: # -- Image repository for the proxy pod. - name: "ghcr.io/liqotech/proxy" + name: "ghcr.io/castai/images/liqo/proxy" # -- Custom version for the proxy image. If not specified, the global tag is used. version: "" service: diff --git a/docs/_downloads/grafana/liqonetwork.json b/docs/_downloads/grafana/liqonetwork.json index fccadd27d0..ff89eb5a96 100644 --- a/docs/_downloads/grafana/liqonetwork.json +++ b/docs/_downloads/grafana/liqonetwork.json @@ -291,7 +291,7 @@ "spec": { "editorMode": "builder", "exemplar": false, - "expr": "liqo_peer_is_connected{cluster_id=\"$clusterid\"}", + "expr": "max by(cluster_id) (liqo_peer_is_connected{cluster_id=\"$clusterid\"})", "format": "table", "instant": false, "interval": "", @@ -2123,6 +2123,56 @@ "refId": "Transmitted", "hidden": false } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(gateway) (rate(liqo_peer_receive_bytes_total{cluster_id=\"$clusterid\"}[$__rate_interval])) * 8", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Received {{gateway}}", + "range": true, + "useBackend": false + } + }, + "refId": "ReceivedGateway", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(gateway) (rate(liqo_peer_transmit_bytes_total{cluster_id=\"$clusterid\"}[$__rate_interval])) * 8", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Transmitted {{gateway}}", + "range": true, + "useBackend": false + } + }, + "refId": "TransmittedGateway", + "hidden": false + } } ], "transformations": [], @@ -2933,10 +2983,10 @@ "spec": { "disableTextWrap": false, "editorMode": "builder", - "expr": "sum(liqo_peer_latency_us{cluster_id=\"$clusterid\"})", + "expr": "sum by(gateway) (liqo_peer_latency_us{cluster_id=\"$clusterid\"})", "fullMetaSearch": false, "includeNullMetadata": true, - "legendFormat": "Latency", + "legendFormat": "{{gateway}}", "range": true, "useBackend": false } @@ -2944,6 +2994,31 @@ "refId": "Latency", "hidden": false } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg(liqo_peer_latency_us{cluster_id=\"$clusterid\"} > 0)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Average", + "range": true, + "useBackend": false + } + }, + "refId": "LatencyAverage", + "hidden": false + } } ], "transformations": [], @@ -2966,7 +3041,7 @@ "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { "hideZeros": false, @@ -2987,8 +3062,7 @@ ] }, "color": { - "mode": "fixed", - "fixedColor": "semi-dark-blue" + "mode": "palette-classic" }, "custom": { "axisBorderShow": false, @@ -3030,6 +3104,117 @@ } } } + }, + "panel-60": { + "kind": "Panel", + "spec": { + "id": 60, + "title": "Gateway connectivity", + "description": "Connection status (1 = connected, 0 = disconnected) for each gateway.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "datasource": { + "name": "${datasource}" + }, + "spec": { + "disableTextWrap": false, + "editorMode": "builder", + "expr": "liqo_peer_is_connected{cluster_id=\"$clusterid\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{gateway}}", + "range": true, + "useBackend": false + } + }, + "refId": "Connectivity", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.1.0", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "min": 0, + "max": 1, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } } }, "layout": { @@ -3291,6 +3476,19 @@ "name": "panel-44" } } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 24, + "width": 24, + "height": 6, + "element": { + "kind": "ElementReference", + "name": "panel-60" + } + } } ] } @@ -3515,4 +3713,4 @@ } ] } -} \ No newline at end of file +} diff --git a/docs/usage/liqoctl/liqoctl_network.md b/docs/usage/liqoctl/liqoctl_network.md index 5565542760..7f204171ed 100644 --- a/docs/usage/liqoctl/liqoctl_network.md +++ b/docs/usage/liqoctl/liqoctl_network.md @@ -50,6 +50,10 @@ liqoctl network connect [flags] >Type of Gateway Client. Leave empty to use default Liqo implementation of WireGuard **(default "networking.liqo.io/v1beta1/wggatewayclienttemplates")** +`--gw-replicas` _int32_: + +>Number of active-active gateway replicas to create **(default 1)** + `--gw-server-service-loadbalancerip` _string_: >Force LoadBalancer IP of the Gateway Server service. Leave empty to use the one provided by the LoadBalancer provider diff --git a/docs/usage/liqoctl/liqoctl_peer.md b/docs/usage/liqoctl/liqoctl_peer.md index ec175845c2..561d25534d 100644 --- a/docs/usage/liqoctl/liqoctl_peer.md +++ b/docs/usage/liqoctl/liqoctl_peer.md @@ -75,6 +75,10 @@ liqoctl peer [flags] >Define the port used by the gateway client to connect to the gateway server. This value overrides the one automatically retrieved by Liqo and it is useful when the server is not directly reachable (e.g. the server is behind a NAT) +`--gw-replicas` _int32_: + +>Number of active-active gateway replicas to create **(default 1)** + `--gw-server-service-loadbalancerip` _string_: >IP of the LoadBalancer for the Gateway Server service diff --git a/docs/usage/liqoctl/liqoctl_test.md b/docs/usage/liqoctl/liqoctl_test.md index 9760ac7c36..332230ca19 100644 --- a/docs/usage/liqoctl/liqoctl_test.md +++ b/docs/usage/liqoctl/liqoctl_test.md @@ -18,7 +18,9 @@ Launch E2E tests for the network Launch network E2E tests. ```{warning} - to run the tests you need to have kyverno installed on every cluster https://kyverno.io/docs/installation/methods/ . + to run the tests you need to have Kyverno v1.13 or later installed on every cluster +(https://kyverno.io/docs/installation/methods/). The tests rely on the policies.kyverno.io/v1 +NamespacedMutatingPolicy API, which is not available in earlier Kyverno versions. ``` This command allows to launch E2E tests, which are used to check the network functionalities between the clusters. The command needs to be run on the cluster that will act as the consumer, @@ -47,6 +49,12 @@ or or +```bash + $ liqoctl test network --remote-kubeconfigs $HOME/.kube/config2,$HOME/.kube/config3 --node-pod +``` + +or + ```bash $ liqoctl test network --remote-kubeconfigs $HOME/.kube/config2,$HOME/.kube/config3 --np-nodes all --np-ext --pod-np ``` @@ -84,6 +92,10 @@ or >Enable curl from external to loadbalancer service +`--node-pod` + +>Enable curl from node to pod + `--np-ext` >Enable curl from external to nodeport service diff --git a/examples/networking/wireguard-client-default.yaml b/examples/networking/wireguard-client-default.yaml index 1f21fd36bf..d30e71cc29 100644 --- a/examples/networking/wireguard-client-default.yaml +++ b/examples/networking/wireguard-client-default.yaml @@ -47,7 +47,7 @@ spec: - --ping-loss-threshold=5 - --ping-interval=2s - --ping-update-status-interval=10s - image: ghcr.io/liqotech/gateway: + image: ghcr.io/castai/images/liqo/gateway: imagePullPolicy: IfNotPresent name: gateway securityContext: @@ -67,7 +67,7 @@ spec: - --endpoint-port={{ .Spec.Endpoint.Port }} - --metrics-address=:8082 - --health-probe-bind-address=:8083 - image: ghcr.io/liqotech/gateway/wireguard: + image: ghcr.io/castai/images/liqo/gateway/wireguard: imagePullPolicy: IfNotPresent name: wireguard securityContext: @@ -90,7 +90,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: ghcr.io/liqotech/gateway/geneve: + image: ghcr.io/castai/images/liqo/gateway/geneve: imagePullPolicy: IfNotPresent name: geneve securityContext: diff --git a/examples/networking/wireguard-client-high-availability.yaml b/examples/networking/wireguard-client-high-availability.yaml index 7d57068407..ee41e2c283 100644 --- a/examples/networking/wireguard-client-high-availability.yaml +++ b/examples/networking/wireguard-client-high-availability.yaml @@ -55,7 +55,7 @@ spec: - --ping-loss-threshold=5 - --ping-interval=2s - --ping-update-status-interval=10s - image: ghcr.io/liqotech/gateway: + image: ghcr.io/castai/images/liqo/gateway: imagePullPolicy: IfNotPresent name: gateway securityContext: @@ -75,7 +75,7 @@ spec: - --endpoint-port={{ .Spec.Endpoint.Port }} - --metrics-address=:8082 - --health-probe-bind-address=:8083 - image: ghcr.io/liqotech/gateway/wireguard: + image: ghcr.io/castai/images/liqo/gateway/wireguard: imagePullPolicy: IfNotPresent name: wireguard securityContext: @@ -98,7 +98,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: ghcr.io/liqotech/gateway/geneve: + image: ghcr.io/castai/images/liqo/gateway/geneve: imagePullPolicy: IfNotPresent name: geneve securityContext: diff --git a/examples/networking/wireguard-client-pingdisabled.yaml b/examples/networking/wireguard-client-pingdisabled.yaml index 1d5c32255a..eddc7bf1b3 100644 --- a/examples/networking/wireguard-client-pingdisabled.yaml +++ b/examples/networking/wireguard-client-pingdisabled.yaml @@ -44,7 +44,7 @@ spec: - --metrics-address=:8082 - --health-probe-bind-address=:8081 - --ping-enabled=false - image: ghcr.io/liqotech/gateway: + image: ghcr.io/castai/images/liqo/gateway: imagePullPolicy: IfNotPresent name: gateway securityContext: @@ -64,7 +64,7 @@ spec: - --endpoint-port={{ .Spec.Endpoint.Port }} - --metrics-address=:8082 - --health-probe-bind-address=:8083 - image: ghcr.io/liqotech/gateway/wireguard: + image: ghcr.io/castai/images/liqo/gateway/wireguard: imagePullPolicy: IfNotPresent name: wireguard securityContext: @@ -87,7 +87,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: ghcr.io/liqotech/gateway/geneve: + image: ghcr.io/castai/images/liqo/gateway/geneve: imagePullPolicy: IfNotPresent name: geneve securityContext: diff --git a/examples/networking/wireguard-client-selector.yaml b/examples/networking/wireguard-client-selector.yaml index f6584a4c89..fc3cb13472 100644 --- a/examples/networking/wireguard-client-selector.yaml +++ b/examples/networking/wireguard-client-selector.yaml @@ -56,7 +56,7 @@ spec: - --ping-loss-threshold=5 - --ping-interval=2s - --ping-update-status-interval=10s - image: ghcr.io/liqotech/gateway: + image: ghcr.io/castai/images/liqo/gateway: imagePullPolicy: IfNotPresent name: gateway securityContext: @@ -76,7 +76,7 @@ spec: - --endpoint-port={{ .Spec.Endpoint.Port }} - --metrics-address=:8082 - --health-probe-bind-address=:8083 - image: ghcr.io/liqotech/gateway/wireguard: + image: ghcr.io/castai/images/liqo/gateway/wireguard: imagePullPolicy: IfNotPresent name: wireguard securityContext: @@ -99,7 +99,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: ghcr.io/liqotech/gateway/geneve: + image: ghcr.io/castai/images/liqo/gateway/geneve: imagePullPolicy: IfNotPresent name: geneve securityContext: diff --git a/examples/networking/wireguard-server-default.yaml b/examples/networking/wireguard-server-default.yaml index d20005abf8..3fffb4dfed 100644 --- a/examples/networking/wireguard-server-default.yaml +++ b/examples/networking/wireguard-server-default.yaml @@ -47,7 +47,7 @@ spec: - --ping-loss-threshold=5 - --ping-interval=2s - --ping-update-status-interval=10s - image: ghcr.io/liqotech/gateway: + image: ghcr.io/castai/images/liqo/gateway: imagePullPolicy: IfNotPresent name: gateway securityContext: @@ -66,7 +66,7 @@ spec: - --listen-port={{ .Spec.Endpoint.Port }} - --metrics-address=:8082 - --health-probe-bind-address=:8083 - image: ghcr.io/liqotech/gateway/wireguard: + image: ghcr.io/castai/images/liqo/gateway/wireguard: imagePullPolicy: IfNotPresent name: wireguard securityContext: @@ -89,7 +89,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: ghcr.io/liqotech/gateway/geneve: + image: ghcr.io/castai/images/liqo/gateway/geneve: imagePullPolicy: IfNotPresent name: geneve securityContext: diff --git a/examples/networking/wireguard-server-high-availability.yaml b/examples/networking/wireguard-server-high-availability.yaml index 40eb5ae8d7..f9a58b57eb 100644 --- a/examples/networking/wireguard-server-high-availability.yaml +++ b/examples/networking/wireguard-server-high-availability.yaml @@ -55,7 +55,7 @@ spec: - --ping-loss-threshold=5 - --ping-interval=2s - --ping-update-status-interval=10s - image: ghcr.io/liqotech/gateway: + image: ghcr.io/castai/images/liqo/gateway: imagePullPolicy: IfNotPresent name: gateway securityContext: @@ -74,7 +74,7 @@ spec: - --listen-port={{ .Spec.Endpoint.Port }} - --metrics-address=:8082 - --health-probe-bind-address=:8083 - image: ghcr.io/liqotech/gateway/wireguard: + image: ghcr.io/castai/images/liqo/gateway/wireguard: imagePullPolicy: IfNotPresent name: wireguard securityContext: @@ -97,7 +97,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: ghcr.io/liqotech/gateway/geneve: + image: ghcr.io/castai/images/liqo/gateway/geneve: imagePullPolicy: IfNotPresent name: geneve securityContext: diff --git a/examples/networking/wireguard-server-overridetargetport.yaml b/examples/networking/wireguard-server-overridetargetport.yaml index 0d5b921316..d97a9396a3 100644 --- a/examples/networking/wireguard-server-overridetargetport.yaml +++ b/examples/networking/wireguard-server-overridetargetport.yaml @@ -47,7 +47,7 @@ spec: - --ping-loss-threshold=5 - --ping-interval=2s - --ping-update-status-interval=10s - image: ghcr.io/liqotech/gateway: + image: ghcr.io/castai/images/liqo/gateway: imagePullPolicy: IfNotPresent name: gateway securityContext: @@ -66,7 +66,7 @@ spec: - --listen-port=30000 - --metrics-address=:8082 - --health-probe-bind-address=:8083 - image: ghcr.io/liqotech/gateway/wireguard: + image: ghcr.io/castai/images/liqo/gateway/wireguard: imagePullPolicy: IfNotPresent name: wireguard securityContext: @@ -89,7 +89,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: ghcr.io/liqotech/gateway/geneve: + image: ghcr.io/castai/images/liqo/gateway/geneve: imagePullPolicy: IfNotPresent name: geneve securityContext: diff --git a/examples/networking/wireguard-server-pingdisabled.yaml b/examples/networking/wireguard-server-pingdisabled.yaml index 9097a91272..b7a31fec0e 100644 --- a/examples/networking/wireguard-server-pingdisabled.yaml +++ b/examples/networking/wireguard-server-pingdisabled.yaml @@ -44,7 +44,7 @@ spec: - --metrics-address=:8082 - --health-probe-bind-address=:8081 - --ping-enabled=false - image: ghcr.io/liqotech/gateway: + image: ghcr.io/castai/images/liqo/gateway: imagePullPolicy: IfNotPresent name: gateway securityContext: @@ -63,7 +63,7 @@ spec: - --listen-port={{ .Spec.Endpoint.Port }} - --metrics-address=:8082 - --health-probe-bind-address=:8083 - image: ghcr.io/liqotech/gateway/wireguard: + image: ghcr.io/castai/images/liqo/gateway/wireguard: imagePullPolicy: IfNotPresent name: wireguard securityContext: @@ -86,7 +86,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: ghcr.io/liqotech/gateway/geneve: + image: ghcr.io/castai/images/liqo/gateway/geneve: imagePullPolicy: IfNotPresent name: geneve securityContext: diff --git a/examples/networking/wireguard-server-selector.yaml b/examples/networking/wireguard-server-selector.yaml index ad59a06c03..ef3765b9e1 100644 --- a/examples/networking/wireguard-server-selector.yaml +++ b/examples/networking/wireguard-server-selector.yaml @@ -56,7 +56,7 @@ spec: - --ping-loss-threshold=5 - --ping-interval=2s - --ping-update-status-interval=10s - image: ghcr.io/liqotech/gateway: + image: ghcr.io/castai/images/liqo/gateway: imagePullPolicy: IfNotPresent name: gateway securityContext: @@ -75,7 +75,7 @@ spec: - --listen-port={{ .Spec.Endpoint.Port }} - --metrics-address=:8082 - --health-probe-bind-address=:8083 - image: ghcr.io/liqotech/gateway/wireguard: + image: ghcr.io/castai/images/liqo/gateway/wireguard: imagePullPolicy: IfNotPresent name: wireguard securityContext: @@ -98,7 +98,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: ghcr.io/liqotech/gateway/geneve: + image: ghcr.io/castai/images/liqo/gateway/geneve: imagePullPolicy: IfNotPresent name: geneve securityContext: diff --git a/go.mod b/go.mod index fce3637fdb..a4d8046e6b 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/subscription/armsubscription v1.2.0 github.com/aws/aws-sdk-go v1.54.6 github.com/go-git/go-git/v5 v5.17.0 - github.com/google/nftables v0.3.0 + github.com/google/nftables v0.3.1-0.20260430172505-f9b52ed2ba65 github.com/google/uuid v1.6.0 github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e github.com/gruntwork-io/terratest v0.48.1 @@ -219,7 +219,7 @@ require ( github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mattn/go-zglob v0.0.3 // indirect github.com/mdlayher/genetlink v1.2.0 // indirect - github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect + github.com/mdlayher/netlink v1.8.1-0.20251028132421-dcc6cab9a6eb // indirect github.com/mdlayher/socket v0.5.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect diff --git a/go.sum b/go.sum index 7427b76f55..77dffc6126 100644 --- a/go.sum +++ b/go.sum @@ -473,8 +473,8 @@ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg= -github.com/google/nftables v0.3.0/go.mod h1:BCp9FsrbF1Fn/Yu6CLUc9GGZFw/+hsxfluNXXmxBfRM= +github.com/google/nftables v0.3.1-0.20260430172505-f9b52ed2ba65 h1:git1jF54pv+jlZY51lOBW93GCdOq1K6ZXOzgyxrFJYM= +github.com/google/nftables v0.3.1-0.20260430172505-f9b52ed2ba65/go.mod h1:rG94e4n789TcldN5fAqkB3qxgOUDo3brNpTlUIo8zgY= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -642,8 +642,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/mdlayher/genetlink v1.2.0 h1:4yrIkRV5Wfk1WfpWTcoOlGmsWgQj3OtQN9ZsbrE+XtU= github.com/mdlayher/genetlink v1.2.0/go.mod h1:ra5LDov2KrUCZJiAtEvXXZBxGMInICMXIwshlJ+qRxQ= github.com/mdlayher/netlink v1.6.0/go.mod h1:0o3PlBmGst1xve7wQ7j/hwpNaFaH4qCRyWCdcZk8/vA= -github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 h1:A1Cq6Ysb0GM0tpKMbdCXCIfBclan4oHk1Jb+Hrejirg= -github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42/go.mod h1:BB4YCPDOzfy7FniQ/lxuYQ3dgmM2cZumHbK8RpTjN2o= +github.com/mdlayher/netlink v1.8.1-0.20251028132421-dcc6cab9a6eb h1:DksQ+rxoqmUG32L6R1IbDH43i/Z9CpwBT6mSc9D9mXs= +github.com/mdlayher/netlink v1.8.1-0.20251028132421-dcc6cab9a6eb/go.mod h1:JVS396/XsyadqrXigRedyb1Q7yEfSq95lheQxgyQ6xA= github.com/mdlayher/socket v0.1.1/go.mod h1:mYV5YIZAfHh4dzDVzI8x8tWLWCliuX8Mon5Awbj+qDs= github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos= github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ= diff --git a/hack/build-images-kind.sh b/hack/build-images-kind.sh new file mode 100755 index 0000000000..55c2368cbd --- /dev/null +++ b/hack/build-images-kind.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +set -e +set -o nounset +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +cd "${REPO_ROOT}" + +# Environment variables respected by build/liqo/build.sh +DOCKER_REGISTRY="${DOCKER_REGISTRY:-us-docker.pkg.dev}" +DOCKER_ORGANIZATION="${DOCKER_ORGANIZATION:-castai-hub/library/liqo}" +DOCKER_TAG="${DOCKER_TAG:-$(git rev-parse HEAD)}" +DOCKER_PUSH="${DOCKER_PUSH:-false}" +ARCHS="${ARCHS:-linux/$(go env GOARCH)}" + +export DOCKER_REGISTRY DOCKER_ORGANIZATION DOCKER_TAG DOCKER_PUSH ARCHS + +# Go components handled by build/liqo/build.sh +GO_COMPONENTS=( + crd-replicator + ipam + liqo-controller-manager + webhook + uninstaller + virtual-kubelet + metric-agent + telemetry + proxy + gateway + gateway/wireguard + gateway/geneve + fabric +) + +# Non-Go components built directly from their Dockerfiles +STANDARD_COMPONENTS=( + cert-creator + liqo-crd-upgrade +) + +image_component_for() { + local component_path="$1" + local basename_component + basename_component=$(basename "${component_path}") + + if [[ "${basename_component}" == "geneve" || "${basename_component}" == "wireguard" ]]; then + echo "gateway/${basename_component}" + else + echo "${basename_component}" + fi +} + +image_tag_for() { + local image_component="$1" + local tag + + if [[ "${DOCKER_TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then + tag="${DOCKER_REGISTRY}/${DOCKER_ORGANIZATION}/${image_component}:${DOCKER_TAG}" + else + tag="${DOCKER_REGISTRY}/${DOCKER_ORGANIZATION}/${image_component}-ci:${DOCKER_TAG}" + fi + + echo "${tag}" +} + +load_into_kind() { + local image_tag="$1" + local clusters + + clusters=$(kind get clusters 2>/dev/null || true) + if [ -z "${clusters}" ]; then + echo "No kind clusters found; image ${image_tag} was built locally only." + return + fi + + for cluster in ${clusters}; do + echo "Loading image ${image_tag} into kind cluster '${cluster}'..." + kind load docker-image "${image_tag}" --name "${cluster}" + done +} + +build_go_component() { + local component="$1" + local component_path="./cmd/${component}" + local image_component + image_component=$(image_component_for "${component_path}") + local image_tag + image_tag=$(image_tag_for "${image_component}") + + echo "============================================" + echo "Building Go component: ${component}" + echo "============================================" + + "${REPO_ROOT}/build/liqo/build.sh" "${component_path}" + + load_into_kind "${image_tag}" +} + +build_standard_component() { + local component="$1" + local dockerfile_path="build/${component}/Dockerfile" + local image_tag + image_tag=$(image_tag_for "${component}") + + echo "============================================" + echo "Building standard component: ${component}" + echo "============================================" + + local platform + platform="linux/$(go env GOARCH)" + + docker buildx build --platform "${platform}" \ + --build-arg COMPONENT="${component}" \ + -t "${image_tag}" \ + -f "./${dockerfile_path}" . \ + --load + + load_into_kind "${image_tag}" +} + +echo "Building all Liqo images and loading them into kind clusters..." +echo "DOCKER_REGISTRY=${DOCKER_REGISTRY}" +echo "DOCKER_ORGANIZATION=${DOCKER_ORGANIZATION}" +echo "DOCKER_TAG=${DOCKER_TAG}" +echo "DOCKER_PUSH=${DOCKER_PUSH}" +echo "ARCHS=${ARCHS}" + +for component in "${GO_COMPONENTS[@]}"; do + build_go_component "${component}" +done + +for component in "${STANDARD_COMPONENTS[@]}"; do + build_standard_component "${component}" +done + +echo "All images built and loaded into kind clusters." diff --git a/hack/multi-gw-debug/README.md b/hack/multi-gw-debug/README.md new file mode 100644 index 0000000000..8f05991083 --- /dev/null +++ b/hack/multi-gw-debug/README.md @@ -0,0 +1,85 @@ +# Multi-Gateway Debugging Layout + +This directory contains a helper that opens an iTerm2 split-pane view and runs +one of the existing tcpdump-based debug scripts on every Liqo gateway pod in two +Kubernetes contexts at once. + +## Requirements + +- **iTerm2** (macOS terminal emulator) with the Python API enabled. +- **Python 3** with the `venv` module available. +- **`kubectl`** configured with access to both clusters/contexts. +- Gateway pods must be labelled with `networking.liqo.io/component: gateway`. + +### Enable the iTerm2 Python API + +1. Open iTerm2. +2. Go to **Preferences** → **General** → **Magic**. +3. Check **Enable Python API**. +4. Restart iTerm2 if prompted. + +## Files + +- `debug-multi-gw.sh` — main entrypoint. Creates a local Python virtual + environment and installs dependencies automatically. +- `iterm-layout.py` — iTerm2 Python API helper that creates the split panes. +- `tcpdump-http-summary.sh` — captures and summarizes HTTP traffic on gateway + pods. +- `tcpdump-traffic-amount.sh` — counts HTTP packets on gateway pods. +- `requirements.txt` — Python dependencies (currently `iterm2`). + +## Usage + +Run the script from a shell inside iTerm2: + +```bash +./hack/multi-gw-debug/debug-multi-gw.sh +./hack/multi-gw-debug/debug-multi-gw.sh custom +``` + +On first run the script creates a Python virtual environment under +`hack/multi-gw-debug/.venv` and installs the dependencies listed in +`requirements.txt`. Subsequent runs reuse the existing virtual environment. + +Supported modes: + +- `http-summary` — runs `tcpdump-http-summary.sh` on each gateway pod. +- `traffic-amount` — runs `tcpdump-traffic-amount.sh` on each gateway pod. +- `custom` — runs the provided shell command in each gateway pod terminal. + +Examples: + +```bash +./hack/multi-gw-debug/debug-multi-gw.sh http-summary kind-cl01 kind-cl02 +./hack/multi-gw-debug/debug-multi-gw.sh custom "tcpdump -i any port 443" kind-cl01 kind-cl02 +``` + +## What it does + +1. Reads the local cluster ID from each context by inspecting the + `liqo-clusterid-configmap` ConfigMap in the `liqo` namespace. +2. Queries both Kubernetes contexts for pods labelled + `networking.liqo.io/component: gateway`, then keeps only the gateway pods + whose Pod spec contains `--remote-cluster-id=`. + This ensures the left column shows only the gateways that peer with + ``, and the right column shows only the gateways that peer with + ``. +3. Uses the iTerm2 Python API to split the current terminal window: + - **Left column**: gateway pods of `` that peer with ``. + - **Right column**: gateway pods of `` that peer with ``. + - Pods are aligned by row so you can easily compare the same logical row + across the two clusters. If the clusters have a different number of + matching gateway pods, the extra pods occupy additional rows in their + column. +4. In each sub-terminal it copies the selected debug script into the pod and + executes it interactively, clearing the terminal first. + +## Notes + +- The script must be launched from within an iTerm2 window so the Python API + can connect to it. +- `kubectl exec -it` is used, so each pane behaves like an interactive shell + session inside the gateway pod. +- The copied script is placed under `/tmp/` inside the pod and executed with + `/bin/sh`, so it does not rely on the executable bit being preserved by + `kubectl cp`. diff --git a/hack/multi-gw-debug/debug-multi-gw.sh b/hack/multi-gw-debug/debug-multi-gw.sh new file mode 100755 index 0000000000..875022c89f --- /dev/null +++ b/hack/multi-gw-debug/debug-multi-gw.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +VENV_DIR="${SCRIPT_DIR}/.venv" +REQUIREMENTS_FILE="${SCRIPT_DIR}/requirements.txt" + +usage() { + cat < + $(basename "$0") custom + +Modes: + http-summary Run tcpdump-http-summary.sh on each gateway pod + traffic-amount Run tcpdump-traffic-amount.sh on each gateway pod + custom Run in every gateway pod terminal + +Examples: + $(basename "$0") http-summary kind-cl01 kind-cl02 + $(basename "$0") custom "tcpdump -i any port 443" kind-cl01 kind-cl02 +EOF +} + +if [ "$#" -ne 3 ] && [ "$#" -ne 4 ]; then + usage >&2 + exit 1 +fi + +MODE="$1" + +if [ "${MODE}" = "custom" ]; then + if [ "$#" -ne 4 ]; then + usage >&2 + exit 1 + fi + CUSTOM_COMMAND="$2" + if [ -z "${CUSTOM_COMMAND}" ]; then + echo "Custom command cannot be empty." >&2 + usage >&2 + exit 1 + fi + CTX1="$3" + CTX2="$4" + TARGET="${CUSTOM_COMMAND}" +else + if [ "$#" -ne 3 ]; then + usage >&2 + exit 1 + fi + CTX1="$2" + CTX2="$3" + + case "${MODE}" in + http-summary) + TARGET="${SCRIPT_DIR}/tcpdump-http-summary.sh" + ;; + traffic-amount) + TARGET="${SCRIPT_DIR}/tcpdump-traffic-amount.sh" + ;; + *) + echo "Unknown mode: ${MODE}" >&2 + usage >&2 + exit 1 + ;; + esac + + if [ ! -f "${TARGET}" ]; then + echo "Target script not found: ${TARGET}" >&2 + exit 1 + fi +fi + +for cmd in kubectl python3; do + if ! command -v "${cmd}" >/dev/null 2>&1; then + echo "Required command not found: ${cmd}" >&2 + exit 1 + fi +done + +if [ ! -d "${VENV_DIR}" ]; then + echo "Creating Python virtual environment in ${VENV_DIR}..." + python3 -m venv "${VENV_DIR}" +fi + +if [ ! -f "${REQUIREMENTS_FILE}" ]; then + echo "Requirements file not found: ${REQUIREMENTS_FILE}" >&2 + exit 1 +fi + +echo "Installing Python dependencies in virtual environment..." +"${VENV_DIR}/bin/pip" install -q -r "${REQUIREMENTS_FILE}" + +get_cluster_id() { + local ctx="$1" + kubectl --context "${ctx}" get configmap liqo-clusterid-configmap -n castai-omni \ + -o jsonpath='{.data.CLUSTER_ID}' 2>/dev/null || true +} + +get_gateway_pods() { + local ctx="$1" + kubectl --context "${ctx}" get pods -A \ + -l networking.liqo.io/component=gateway \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{"/"}{.metadata.name}{"\n"}{end}' 2>/dev/null || true +} + +filter_pods_by_remote_cluster() { + local ctx="$1" + local remote_cluster_id="$2" + local pods="$3" + local pod ns name + + if [ -z "${remote_cluster_id}" ]; then + echo "${pods}" + return + fi + + while IFS= read -r pod; do + [ -z "${pod}" ] && continue + ns="${pod%/*}" + name="${pod#*/}" + if kubectl --context "${ctx}" -n "${ns}" get pod "${name}" -o yaml 2>/dev/null | \ + grep -q -- "--remote-cluster-id=${remote_cluster_id}"; then + echo "${pod}" + fi + done <<< "${pods}" +} + +CLUSTER_ID1="$(get_cluster_id "${CTX1}")" +CLUSTER_ID2="$(get_cluster_id "${CTX2}")" + +if [ -z "${CLUSTER_ID1}" ]; then + echo "Warning: could not read cluster ID for context ${CTX1} from configmap castai-omni/liqo-clusterid-configmap" >&2 +fi +if [ -z "${CLUSTER_ID2}" ]; then + echo "Warning: could not read cluster ID for context ${CTX2} from configmap castai-omni/liqo-clusterid-configmap" >&2 +fi + +PODS1_RAW="$(get_gateway_pods "${CTX1}")" +PODS2_RAW="$(get_gateway_pods "${CTX2}")" + +PODS1_RAW="$(filter_pods_by_remote_cluster "${CTX1}" "${CLUSTER_ID2}" "${PODS1_RAW}")" +PODS2_RAW="$(filter_pods_by_remote_cluster "${CTX2}" "${CLUSTER_ID1}" "${PODS2_RAW}")" + +if [ -z "${PODS1_RAW}" ] && [ -z "${PODS2_RAW}" ]; then + echo "No gateway pods found for the cross-cluster pair (label: networking.liqo.io/component=gateway, remote-cluster-id points to the other context)." >&2 + exit 1 +fi + +PODS1_LIST="$(echo "${PODS1_RAW}" | grep -v '^$' | paste -sd ',' -)" +PODS2_LIST="$(echo "${PODS2_RAW}" | grep -v '^$' | paste -sd ',' -)" + +echo "Context ${CTX1} (${CLUSTER_ID1:-unknown}): $(echo "${PODS1_RAW}" | grep -c '^.') matching gateway pod(s)" +echo "Context ${CTX2} (${CLUSTER_ID2:-unknown}): $(echo "${PODS2_RAW}" | grep -c '^.') matching gateway pod(s)" +echo "Launching iTerm2 layout..." + +exec "${VENV_DIR}/bin/python3" "${SCRIPT_DIR}/iterm-layout.py" \ + "${MODE}" \ + "${CTX1}" \ + "${CTX2}" \ + "${PODS1_LIST}" \ + "${PODS2_LIST}" \ + "${TARGET}" diff --git a/hack/multi-gw-debug/iterm-layout.py b/hack/multi-gw-debug/iterm-layout.py new file mode 100755 index 0000000000..97fc9d5274 --- /dev/null +++ b/hack/multi-gw-debug/iterm-layout.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""iTerm2 Python API helper for multi-gateway debugging layout. + +This script is invoked by debug-multi-gw.sh. It splits the current iTerm2 +window into two columns (one per Kubernetes context) and runs the requested +debugging script on every gateway pod, aligned by row. +""" + +import asyncio +import os +import shlex +import sys + +import iterm2 + + +def usage() -> None: + print( + "Usage: iterm-layout.py ", + file=sys.stderr, + ) + print(" mode: http-summary | traffic-amount | custom", file=sys.stderr) + print(" target: script path for built-in modes, shell command for custom", file=sys.stderr) + print(" pods1/pods2 are comma-separated lists of namespace/name values.", file=sys.stderr) + + +def parse_pods(value: str) -> list[str]: + return [p for p in value.split(",") if p] + + +def build_command(ctx: str, pod: str, mode: str, target: str) -> str: + ns, name = pod.split("/", 1) + + # Clear the terminal (fallback to ANSI escape if clear is unavailable). + clear_command = "clear 2>/dev/null || printf '\\033[2J\\033[H'" + + if mode == "custom": + inner_command = f"{clear_command}; {target}" + copy_prefix = "" + else: + script_name = os.path.basename(target) + remote_path = f"/tmp/{script_name}" + inner_command = f"{clear_command}; sh {remote_path}" + copy_prefix = ( + f"kubectl --context {shlex.quote(ctx)} cp {shlex.quote(target)} " + f"{shlex.quote(ns)}/{shlex.quote(name)}:{remote_path} && " + ) + + return ( + f"{copy_prefix}" + f"kubectl --context {shlex.quote(ctx)} -n {shlex.quote(ns)} " + f"exec -it {shlex.quote(name)} -- " + f"/bin/sh -c {shlex.quote(inner_command)}\n" + ) + + +async def build_grid( + tab: iterm2.Tab, + pods1: list[str], + pods2: list[str], +) -> tuple[list[iterm2.Session], list[iterm2.Session]]: + """Create a two-column grid of sessions aligned by row. + + The algorithm builds the layout row-by-row to avoid splitting the same pane + in two different directions, which can confuse iTerm2's splitter tree. + """ + sessions_col1: list[iterm2.Session] = [] + sessions_col2: list[iterm2.Session] = [] + + # Use the current session for column 1 if available; otherwise use it for + # column 2 so we never leave the original pane empty. + if pods1: + sessions_col1.append(tab.current_session) + elif pods2: + sessions_col2.append(tab.current_session) + + # Create the first row of column 2 by splitting the first column 1 pane + # vertically. From this point on each pane is only split horizontally. + if pods1 and pods2: + first_col2 = await sessions_col1[0].async_split_pane(vertical=True) + sessions_col2.append(first_col2) + await asyncio.sleep(0.1) + + max_rows = max(len(pods1), len(pods2)) + for _row in range(1, max_rows): + if len(sessions_col1) < len(pods1): + new_session = await sessions_col1[-1].async_split_pane(vertical=False) + sessions_col1.append(new_session) + if len(sessions_col2) < len(pods2): + new_session = await sessions_col2[-1].async_split_pane(vertical=False) + sessions_col2.append(new_session) + await asyncio.sleep(0.05) + + return sessions_col1, sessions_col2 + + +async def main(connection: iterm2.Connection) -> None: + if len(sys.argv) < 7: + usage() + sys.exit(1) + + _mode = sys.argv[1] + mode = sys.argv[1] + ctx1 = sys.argv[2] + ctx2 = sys.argv[3] + pods1 = parse_pods(sys.argv[4]) + pods2 = parse_pods(sys.argv[5]) + target = sys.argv[6] + + if not pods1 and not pods2: + print("No gateway pods to debug.", file=sys.stderr) + sys.exit(1) + + app = await iterm2.async_get_app(connection) + window = app.current_terminal_window + if window is None: + print("No current iTerm2 terminal window found.", file=sys.stderr) + sys.exit(1) + + tab = window.current_tab + if tab is None: + print("No current tab found in iTerm2 window.", file=sys.stderr) + sys.exit(1) + + sessions_col1, sessions_col2 = await build_grid(tab, pods1, pods2) + + # Label each pane so it is easy to tell which pod it belongs to. + for i, pod in enumerate(pods1): + try: + await sessions_col1[i].async_set_name(f"{ctx1}: {pod}") + except Exception: + pass + for i, pod in enumerate(pods2): + try: + await sessions_col2[i].async_set_name(f"{ctx2}: {pod}") + except Exception: + pass + + # Let the layout settle before typing commands. + await asyncio.sleep(0.5) + + for i, pod in enumerate(pods1): + await sessions_col1[i].async_send_text(build_command(ctx1, pod, mode, target)) + + for i, pod in enumerate(pods2): + await sessions_col2[i].async_send_text(build_command(ctx2, pod, mode, target)) + + +iterm2.run_until_complete(main) diff --git a/hack/multi-gw-debug/requirements.txt b/hack/multi-gw-debug/requirements.txt new file mode 100644 index 0000000000..144f789134 --- /dev/null +++ b/hack/multi-gw-debug/requirements.txt @@ -0,0 +1 @@ +iterm2 diff --git a/hack/multi-gw-debug/tcpdump-http-summary.sh b/hack/multi-gw-debug/tcpdump-http-summary.sh new file mode 100755 index 0000000000..3b25315f1d --- /dev/null +++ b/hack/multi-gw-debug/tcpdump-http-summary.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +# Capture data packets (PSH flag set) on port 80 in both directions. +# A packet going *to* port 80 is a request, a packet coming *from* port 80 is a response. +# +# When capturing on '-i any', tcpdump prefixes lines with 'In' or 'Out'. +# We use sed to extract src/dst reliably regardless of that extra field. +tcpdump -l -n -i any 'tcp port 80 and tcp[13] & 8 != 0' 2>/dev/null | while read -r line; do + src=$(echo "$line" | sed -n 's/.* IP \([^ ]*\) > \([^:]*\):.*/\1/p') + dst=$(echo "$line" | sed -n 's/.* IP \([^ ]*\) > \([^:]*\):.*/\2/p') + ts=$(echo "$line" | awk '{print $1}') + + dst_port=${dst##*.} + + if [ "$dst_port" = "80" ]; then + direction="request" + else + direction="response" + fi + + echo "HTTP $direction at $ts ($src -> $dst)" +done \ No newline at end of file diff --git a/hack/multi-gw-debug/tcpdump-traffic-amount.sh b/hack/multi-gw-debug/tcpdump-traffic-amount.sh new file mode 100755 index 0000000000..1bee0e0faf --- /dev/null +++ b/hack/multi-gw-debug/tcpdump-traffic-amount.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +# File holding the running total. +countfile=$(mktemp) +echo 0 >"$countfile" + +# Background timer: every second clear the screen and print the current total. +print_count() { + while true; do + sleep 1 + clear + echo "Total HTTP packets: $(cat "$countfile")" + done +} + +print_count & +timer_pid=$! + +# Ensure the counter file and the background timer are cleaned up on exit. +cleanup() { + rm -f "$countfile" "$countfile.new" + kill "$timer_pid" 2>/dev/null || true +} +trap 'cleanup' INT TERM EXIT + +# Continuously capture HTTP data packets and atomically increment the counter. +tcpdump -l -n -i any 'tcp port 80 and tcp[13] & 8 != 0' 2>/dev/null | while read -r _; do + old=$(cat "$countfile") + new=$((old + 1)) + printf '%s\n' "$new" >"$countfile.new" + mv "$countfile.new" "$countfile" +done \ No newline at end of file diff --git a/pkg/consts/annotations.go b/pkg/consts/annotations.go index 72be5cee3d..ad280f8ecf 100644 --- a/pkg/consts/annotations.go +++ b/pkg/consts/annotations.go @@ -58,4 +58,27 @@ const ( // deployment (and its supporting resources) must not be created by the controller, // as it is managed externally. SkipVkDeploymentAnnotation = "liqo.io/skip-vk-deployment" + + // ProvisionPVCOnAllEdgesAnnotationKey by default a PVC with the Liqo storage class is bound to the virtual node, + // where the pod is scheduled. If this annotation is set to "true", the PVC can be provisioned on all edge nodes. + // WARNING: each edge node will maintain its own version of the PVC. This is useful for avoiding + // strict binding between a PVC and a specific cluster ID. + ProvisionPVCOnAllEdgesAnnotationKey = "omni.cast.ai/provision-on-all-edge-nodes" + + // ProvisionPVCOnAllEdgesAnnotationValue is the value to set on the ProvisionPVCOnAllEdgesAnnotationKey annotation + // to enable the provisioning of a PVC on all edge nodes. + ProvisionPVCOnAllEdgesAnnotationValue = "true" + + // TemplateNameAnnotationKey is the annotation used to stamp the source template + // name on gateway Deployments. It is used by the serialization gate to consider + // only peers that share the same template. + TemplateNameAnnotationKey = "networking.liqo.io/template-name" + // TemplateNamespaceAnnotationKey is the annotation used to stamp the source + // template namespace on gateway Deployments. Template names are not unique + // across namespaces. + TemplateNamespaceAnnotationKey = "networking.liqo.io/template-namespace" + // TemplateGenerationAnnotationKey is the annotation used to stamp the source + // template generation on gateway Deployments. It is used by the serialization + // gate to decide which peers still need to roll out. + TemplateGenerationAnnotationKey = "networking.liqo.io/template-generation" ) diff --git a/pkg/consts/controllers.go b/pkg/consts/controllers.go index 08ad695122..d68063be8a 100644 --- a/pkg/consts/controllers.go +++ b/pkg/consts/controllers.go @@ -80,6 +80,5 @@ const ( CtrlResourceSliceQuotaCreator = "resourceslice_quotacreator" CtrlResourceSliceVNCreator = "resourceslice_vncreator" CtrlPodIPMapping = "pod_ipmapping" - CtrlConfigurationIPMapping = "configuration_ipmapping" CtrlEndpointSlice = "endpointslice" ) diff --git a/pkg/consts/internalnetwork.go b/pkg/consts/internalnetwork.go index a911915b7b..bdd026747c 100644 --- a/pkg/consts/internalnetwork.go +++ b/pkg/consts/internalnetwork.go @@ -30,4 +30,7 @@ const ( // InternalFabricGeneveTunnelFinalizer is the finalizer used to ensure that the geneve tunnel is deleted and the // id is freed. InternalFabricGeneveTunnelFinalizer = "networking.liqo.io/internal-fabric-geneve-tunnel-finalizer" + // ECMPReplicaMarkBase is the base value used for policy-routing marks associated to gateway replicas. + // The mark for replica i is ECMPReplicaMarkBase + i. + ECMPReplicaMarkBase = 0x10000 ) diff --git a/pkg/consts/namespace_mapping.go b/pkg/consts/namespace_mapping.go index 3f1dcfada9..97ddcf94ac 100644 --- a/pkg/consts/namespace_mapping.go +++ b/pkg/consts/namespace_mapping.go @@ -17,6 +17,8 @@ package consts const ( // RemoteClusterID is used to obtain cluster-id from different Liqo resources. RemoteClusterID = "liqo.io/remote-cluster-id" + // EdgeLocationName is the key of edge location name. + EdgeLocationName = "omni.cast.ai/edge-location-name" // TypeLabel is the key of a Liqo label that identifies different types of nodes. // todo: change to NodeTypeLabel TypeLabel = "liqo.io/type" @@ -35,6 +37,10 @@ const ( // RemoteNamespaceManagedByAnnotationKey is the annotation that identifies the NamespaceMap managing a given remote namespace. RemoteNamespaceManagedByAnnotationKey = "liqo.io/managed-by-namespace-map" + // ManagedNamespaceMapAnnotKey is the annotation key that identifies NamespaceMaps whose lifecycle is managed by the user and not by Liqo. + ManagedNamespaceMapAnnotKey = "liqo.io/managed-namespace-map" + // ManagedNamespaceMapAnnotValue is the annotation value that identifies NamespaceMaps whose lifecycle is managed by the user and not by Liqo. + ManagedNamespaceMapAnnotValue = "true" // RemoteNamespaceOriginalNameAnnotationKey is the annotation that identifies the original name of a remote namespace. RemoteNamespaceOriginalNameAnnotationKey = "liqo.io/original-name" // RemoteNamespaceClusterRoleName is the name of the cluster role used to grant permissions to the virtual kubelet in remote namespaces. diff --git a/pkg/consts/storage.go b/pkg/consts/storage.go index 6ca28c5230..a70f0f9625 100644 --- a/pkg/consts/storage.go +++ b/pkg/consts/storage.go @@ -28,4 +28,7 @@ const ( // StorageNamespaceLabel is the label used to mark the liqo storage namespace. StorageNamespaceLabel = "liqo.io/storage-provisioner" + + // OmniStorageClassPrefix is the prefix used to mark OMNI storage classes. + OmniStorageClassPrefix = "omni-" ) diff --git a/pkg/consts/webhook.go b/pkg/consts/webhook.go index fb595eb60e..6f42bb082a 100644 --- a/pkg/consts/webhook.go +++ b/pkg/consts/webhook.go @@ -17,7 +17,7 @@ package consts const ( // VirtualNodeTolerationKey all Pods that have to be scheduled on virtual nodes must have this toleration // to Liqo taint. - VirtualNodeTolerationKey = "virtual-node.liqo.io/not-allowed" + VirtualNodeTolerationKey = "virtual-node.omni.cast.ai/not-allowed" // WebHookLabel used to mark the resouces related to the Liqo webhooks. WebHookLabel = "liqo.io/webhook" diff --git a/pkg/fabric/flags.go b/pkg/fabric/flags.go index 68907d8fbd..c9f079d492 100644 --- a/pkg/fabric/flags.go +++ b/pkg/fabric/flags.go @@ -47,6 +47,9 @@ const ( // FlagNameEnableRouteMonitor is the flag to enable the route monitor. FlagNameEnableRouteMonitor FlagName = "enable-route-monitor" + // FlagNameEnableMultipathHashPolicy enables 5-tuple hashing to improve ECMP load balancing. + FlagNameEnableMultipathHashPolicy FlagName = "enable-multipath-hash-policy" + // FlagNameDisableKernelVersionCheck is the flag to enable the kernel version check. FlagNameDisableKernelVersionCheck FlagName = "disable-kernel-version-check" // FlagNameMinimumKernelVersion is the minimum kernel version required to run the wireguard interface. @@ -81,6 +84,9 @@ func InitFlags(flagset *pflag.FlagSet, opts *Options) { flagset.BoolVar(&opts.EnableNftMonitor, FlagNameEnableNftMonitor.String(), true, "Enable nftables monitor") flagset.BoolVar(&opts.EnableRouteMonitor, FlagNameEnableRouteMonitor.String(), true, "Enable route monitor") + flagset.BoolVar(&opts.EnableMultipathHashPolicy, FlagNameEnableMultipathHashPolicy.String(), false, + "Set fib_multipath_hash_policy=1 to use 5-tuple hashing for multipath routing") + flagset.BoolVar(&opts.DisableKernelVersionCheck, FlagNameDisableKernelVersionCheck.String(), false, "Disable the kernel version check") flagset.Var(&opts.MinimumKernelVersion, string(FlagNameMinimumKernelVersion), "Minimum kernel version required to run the wireguard interface") diff --git a/pkg/fabric/genevetunnel_controller.go b/pkg/fabric/genevetunnel_controller.go index a9111b4e02..838a410cdc 100644 --- a/pkg/fabric/genevetunnel_controller.go +++ b/pkg/fabric/genevetunnel_controller.go @@ -194,7 +194,7 @@ func (r *GeneveTunnelReconciler) Reconcile(ctx context.Context, req ctrl.Request cc := r.connChecker.Load() - if err := r.ensureSender(cc, req.NamespacedName, gt, &internalfabric, &internalnode); err != nil { + if err := r.ensureSender(cc, req.NamespacedName, gt, &internalfabric); err != nil { return ctrl.Result{}, err } @@ -225,7 +225,7 @@ func (r *GeneveTunnelReconciler) Reconcile(ctx context.Context, req ctrl.Request // ensureSender adds the conncheck sender for the given GeneveTunnel if it isn't already // running. It is a no-op after the first successful call. func (r *GeneveTunnelReconciler) ensureSender(cc *conncheck.ConnChecker, key types.NamespacedName, - gt *networkingv1beta1.GeneveTunnel, internalfabric *networkingv1beta1.InternalFabric, internalnode *networkingv1beta1.InternalNode) error { + gt *networkingv1beta1.GeneveTunnel, internalfabric *networkingv1beta1.InternalFabric) error { if cc.HasSender(gt.Name) { return nil } diff --git a/pkg/fabric/options.go b/pkg/fabric/options.go index 967c23b97a..31334bef1e 100644 --- a/pkg/fabric/options.go +++ b/pkg/fabric/options.go @@ -31,8 +31,9 @@ type Options struct { ReconcileTimeout time.Duration - EnableNftMonitor bool - EnableRouteMonitor bool + EnableNftMonitor bool + EnableRouteMonitor bool + EnableMultipathHashPolicy bool DisableKernelVersionCheck bool MinimumKernelVersion kernelversion.KernelVersion diff --git a/pkg/fabric/source-detector/gateway_controller.go b/pkg/fabric/source-detector/gateway_controller.go index 57c3d993fc..14033bc8fd 100644 --- a/pkg/fabric/source-detector/gateway_controller.go +++ b/pkg/fabric/source-detector/gateway_controller.go @@ -71,6 +71,11 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, fmt.Errorf("unable to get the gateway pod %q: %w", req.NamespacedName, err) } + if !pod.DeletionTimestamp.IsZero() { + klog.V(6).Infof("Gateway pod %s is being deleted. Nothing to do", req.String()) + return ctrl.Result{}, nil + } + if pod.Status.PodIP == "" { // If the pod does not have an IP address, we cannot determine the source IP for the internal node. // It will be reconciled again when the IP becomes available. diff --git a/pkg/firewall/firewallconfiguration_controller.go b/pkg/firewall/firewallconfiguration_controller.go index 2be53b2873..5a19c6f751 100644 --- a/pkg/firewall/firewallconfiguration_controller.go +++ b/pkg/firewall/firewallconfiguration_controller.go @@ -116,9 +116,10 @@ func (r *FirewallConfigurationReconciler) Reconcile(ctx context.Context, req ctr klog.V(4).Infof("Reconciling firewallconfiguration %s", req.String()) - defer func() { + //nolint:gocritic // keeping commented code for reference + /* defer func() { err = r.UpdateStatus(ctx, r.EventsRecorder, fwcfg, r.PodName, err) - }() + }() */ // Manage Finalizers and Table deletion. // In nftables, table deletion automatically delete contained chains and rules. diff --git a/pkg/firewall/utils/filterule.go b/pkg/firewall/utils/filterule.go index 93ff2fc746..656491ffca 100644 --- a/pkg/firewall/utils/filterule.go +++ b/pkg/firewall/utils/filterule.go @@ -16,6 +16,7 @@ package utils import ( "fmt" + "math" "strconv" "github.com/google/nftables" @@ -58,15 +59,6 @@ func (fr *FilterRuleWrapper) Add(nftconn *nftables.Conn, chain *nftables.Chain) func (fr *FilterRuleWrapper) Equal(currentrule *nftables.Rule) bool { currentrule.Chain.Table = currentrule.Table newrule, err := forgeFilterRule(fr.FilterRule, currentrule.Chain) - // TODO: this ugly exception is caused by an error in the expr retrieved by nftables library. - // In particular, the expr retrieved by the library when the action is ctmark - // Retrieved expr: &{0 false 3} - // Generated expr: &{1 true 3} - // We think that this error should be caused by a library bug. - // We are going to investigate it further. - if fr.FilterRule.Action == firewallv1beta1.ActionCtMark { - return true - } if err != nil { return false } @@ -112,6 +104,10 @@ func forgeFilterRule(fr *firewallv1beta1.FilterRule, chain *nftables.Chain) (*nf applyRejectAction(rule) case firewallv1beta1.ActionNotrack: applyNotrackAction(rule) + case firewallv1beta1.ActionSetMetaMark: + if err := applySetMetaMarkAction(fr.Value, rule); err != nil { + return nil, fmt.Errorf("cannot apply setmetamark action: %w", err) + } default: } @@ -119,10 +115,16 @@ func forgeFilterRule(fr *firewallv1beta1.FilterRule, chain *nftables.Chain) (*nf } func applyCtMarkAction(value *string, rule *nftables.Rule) error { + if value == nil { + return fmt.Errorf("value is required for %s action", firewallv1beta1.ActionCtMark) + } valueInt, err := strconv.Atoi(*value) if err != nil { return fmt.Errorf("cannot convert value to int: %w", err) } + if valueInt < 0 || valueInt > math.MaxUint32 { + return fmt.Errorf("value %d is out of range for a uint32 mark", valueInt) + } rule.Exprs = append(rule.Exprs, &expr.Immediate{ Register: 1, @@ -244,6 +246,31 @@ func applyNotrackAction(rule *nftables.Rule) { rule.Exprs = append(rule.Exprs, &expr.Notrack{}) } +func applySetMetaMarkAction(value *string, rule *nftables.Rule) error { + if value == nil { + return fmt.Errorf("value is required for %s action", firewallv1beta1.ActionSetMetaMark) + } + valueInt, err := strconv.Atoi(*value) + if err != nil { + return fmt.Errorf("cannot convert value to int: %w", err) + } + if valueInt < 0 || valueInt > math.MaxUint32 { + return fmt.Errorf("value %d is out of range for a uint32 mark", valueInt) + } + rule.Exprs = append(rule.Exprs, + &expr.Immediate{ + Register: 1, + Data: binaryutil.NativeEndian.PutUint32(uint32(valueInt)), + }, + &expr.Meta{ + Key: expr.MetaKeyMARK, + SourceRegister: true, + Register: 1, + }, + ) + return nil +} + func applyCounter(rule *nftables.Rule) { rule.Exprs = append(rule.Exprs, &expr.Counter{Bytes: 1, Packets: 1}) } diff --git a/pkg/firewall/utils/filterule_test.go b/pkg/firewall/utils/filterule_test.go index c3e79924f0..8112f82dfc 100644 --- a/pkg/firewall/utils/filterule_test.go +++ b/pkg/firewall/utils/filterule_test.go @@ -18,6 +18,7 @@ package utils import ( "github.com/google/nftables" + "github.com/google/nftables/binaryutil" "github.com/google/nftables/expr" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -490,5 +491,118 @@ var _ = Describe("FilterRuleWrapper", func() { _, err := forgeFilterRule(fr, chain) Expect(err).NotTo(HaveOccurred()) }) + + It("should error on nil CtMark value", func() { + fr := &firewallv1beta1.FilterRule{ + Name: ptr.To("nil-ctmark"), + Action: firewallv1beta1.ActionCtMark, + Value: nil, + } + _, err := forgeFilterRule(fr, chain) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("value is required for ctmark action")) + }) + + It("should error on nil SetMetaMark value", func() { + fr := &firewallv1beta1.FilterRule{ + Name: ptr.To("nil-setmetamark"), + Action: firewallv1beta1.ActionSetMetaMark, + Value: nil, + } + _, err := forgeFilterRule(fr, chain) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("value is required for setmetamark action")) + }) + + It("should error on invalid SetMetaMark value", func() { + fr := &firewallv1beta1.FilterRule{ + Name: ptr.To("invalid-setmetamark"), + Action: firewallv1beta1.ActionSetMetaMark, + Value: ptr.To("not-a-number"), + } + _, err := forgeFilterRule(fr, chain) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("cannot apply setmetamark action")) + }) + + It("should error on out-of-range CtMark value", func() { + fr := &firewallv1beta1.FilterRule{ + Name: ptr.To("overflow-ctmark"), + Action: firewallv1beta1.ActionCtMark, + Value: ptr.To("4294967296"), + } + _, err := forgeFilterRule(fr, chain) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("out of range")) + }) + + It("should error on negative SetMetaMark value", func() { + fr := &firewallv1beta1.FilterRule{ + Name: ptr.To("negative-setmetamark"), + Action: firewallv1beta1.ActionSetMetaMark, + Value: ptr.To("-1"), + } + _, err := forgeFilterRule(fr, chain) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("out of range")) + }) + }) + + Context("Mark actions", func() { + It("should forge a valid setmetamark rule", func() { + fr := &firewallv1beta1.FilterRule{ + Name: ptr.To("setmetamark-rule"), + Action: firewallv1beta1.ActionSetMetaMark, + Value: ptr.To("65280"), + } + rule, err := forgeFilterRule(fr, chain) + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Exprs).To(HaveLen(2)) + + imm, ok := rule.Exprs[0].(*expr.Immediate) + Expect(ok).To(BeTrue()) + Expect(imm.Register).To(Equal(uint32(1))) + Expect(imm.Data).To(Equal(binaryutil.NativeEndian.PutUint32(65280))) + + meta, ok := rule.Exprs[1].(*expr.Meta) + Expect(ok).To(BeTrue()) + Expect(meta.Key).To(Equal(expr.MetaKeyMARK)) + Expect(meta.SourceRegister).To(BeTrue()) + Expect(meta.Register).To(Equal(uint32(1))) + }) + + It("should forge a valid ctmark rule", func() { + fr := &firewallv1beta1.FilterRule{ + Name: ptr.To("ctmark-rule"), + Action: firewallv1beta1.ActionCtMark, + Value: ptr.To("100"), + } + rule, err := forgeFilterRule(fr, chain) + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Exprs).To(HaveLen(2)) + + imm, ok := rule.Exprs[0].(*expr.Immediate) + Expect(ok).To(BeTrue()) + Expect(imm.Register).To(Equal(uint32(1))) + Expect(imm.Data).To(Equal(binaryutil.NativeEndian.PutUint32(100))) + + ct, ok := rule.Exprs[1].(*expr.Ct) + Expect(ok).To(BeTrue()) + Expect(ct.Key).To(Equal(expr.CtKeyMARK)) + Expect(ct.SourceRegister).To(BeTrue()) + Expect(ct.Register).To(Equal(uint32(1))) + }) + + It("should accept boundary mark values", func() { + for _, v := range []string{"0", "4294967295"} { + fr := &firewallv1beta1.FilterRule{ + Name: ptr.To("boundary-mark"), + Action: firewallv1beta1.ActionSetMetaMark, + Value: ptr.To(v), + } + _, err := forgeFilterRule(fr, chain) + Expect(err).NotTo(HaveOccurred()) + } + }) }) }) diff --git a/pkg/firewall/utils/match.go b/pkg/firewall/utils/match.go index c16cfb788a..d540463741 100644 --- a/pkg/firewall/utils/match.go +++ b/pkg/firewall/utils/match.go @@ -219,6 +219,16 @@ func applyMatchDev(m *firewallv1beta1.Match, rule *nftables.Rule, op expr.CmpOp) return err } + var data []byte + if m.Dev.Wildcard { + // Prefix match: compare only the prefix bytes without null-padding to 16 bytes. + // nftables performs a byte-by-byte comparison, so a short data slice matches any + // interface name that starts with the given prefix. + data = []byte(m.Dev.Value) + } else { + data = ifname(m.Dev.Value) + } + rule.Exprs = append(rule.Exprs, &expr.Meta{ Register: 1, @@ -227,7 +237,7 @@ func applyMatchDev(m *firewallv1beta1.Match, rule *nftables.Rule, op expr.CmpOp) &expr.Cmp{ Op: op, Register: 1, - Data: ifname(m.Dev.Value), + Data: data, }, ) diff --git a/pkg/firewall/utils/match_test.go b/pkg/firewall/utils/match_test.go index 294eb49ba1..b83465350a 100644 --- a/pkg/firewall/utils/match_test.go +++ b/pkg/firewall/utils/match_test.go @@ -18,6 +18,7 @@ package utils import ( "github.com/google/nftables" + "github.com/google/nftables/expr" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -253,6 +254,47 @@ var _ = Describe("Match Functions", func() { Expect(rule.Exprs).NotTo(BeEmpty()) }) + It("should apply dev match with wildcard prefix", func() { + match := &firewallv1beta1.Match{ + Op: firewallv1beta1.MatchOperationEq, + Dev: &firewallv1beta1.MatchDev{ + Value: "liqo.", + Position: firewallv1beta1.MatchDevPositionIn, + Wildcard: true, + }, + } + err := applyMatch(match, rule) + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Exprs).To(HaveLen(2)) + + meta, ok := rule.Exprs[0].(*expr.Meta) + Expect(ok).To(BeTrue()) + Expect(meta.Key).To(Equal(expr.MetaKeyIIFNAME)) + + cmp, ok := rule.Exprs[1].(*expr.Cmp) + Expect(ok).To(BeTrue()) + // The wildcard match compares only the prefix bytes, without null-padding to 16 bytes. + Expect(cmp.Data).To(Equal([]byte("liqo."))) + }) + + It("should null-pad dev match without wildcard", func() { + match := &firewallv1beta1.Match{ + Op: firewallv1beta1.MatchOperationEq, + Dev: &firewallv1beta1.MatchDev{ + Value: "liqo.", + Position: firewallv1beta1.MatchDevPositionIn, + }, + } + err := applyMatch(match, rule) + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Exprs).To(HaveLen(2)) + + cmp, ok := rule.Exprs[1].(*expr.Cmp) + Expect(ok).To(BeTrue()) + Expect(cmp.Data).To(HaveLen(16)) + Expect(cmp.Data).To(Equal(ifname("liqo."))) + }) + It("should apply combined match (proto + IP + port + dev)", func() { matches := []firewallv1beta1.Match{ { diff --git a/pkg/gateway/concurrent/const.go b/pkg/gateway/concurrent/const.go deleted file mode 100644 index c6c6c5efb3..0000000000 --- a/pkg/gateway/concurrent/const.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2019-2026 The Liqo Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package concurrent - -// UnixSocketPath is the path of the Unix socket. -const unixSocketPath string = "/ipc/leader.sock" diff --git a/pkg/gateway/concurrent/doc.go b/pkg/gateway/concurrent/doc.go deleted file mode 100644 index 6d64d1f89a..0000000000 --- a/pkg/gateway/concurrent/doc.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2019-2026 The Liqo Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package concurrent contains the logic to manage same gateway replicas. -// They are managed using an active/passive approach. -// The gateway container try to acquire the active role using the controller manager lease. -// Then, the active gateway is labeled with the ActiveGatewayKey and ActiveGatewayValue, and the passive gateways are unlabeled. -// The gateway service target the active gateway using the ActiveGatewayKey and ActiveGatewayValue labels. -// In order to cohordinate the sidecar containers, the gateway uses a unix socket to manage the IPC, and to start the sidecars when it becomes leader. -package concurrent diff --git a/pkg/gateway/concurrent/gateway.go b/pkg/gateway/concurrent/gateway.go deleted file mode 100644 index 0e198a52ef..0000000000 --- a/pkg/gateway/concurrent/gateway.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2019-2026 The Liqo Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package concurrent - -import ( - "context" - "fmt" - "net" - - corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/manager" - - "github.com/liqotech/liqo/pkg/utils/ipc" -) - -var _ manager.Runnable = &RunnableGateway{} - -// RunnableGateway is a RunnableGateway that manages concurrency. -type RunnableGateway struct { - Client client.Client - - PodName string - GatewayName string - Namespace string - - Socket net.Listener - GuestConnections ipc.GuestConnections -} - -// NewRunnableGatewayStartup creates a new Runnable. -func NewRunnableGatewayStartup(cl client.Client, podName, gatewayName, namespace string, containerNames []string) (*RunnableGateway, error) { - guestConnections := ipc.NewGuestConnections(containerNames) - - socket, err := ipc.CreateListenSocket(unixSocketPath) - if err != nil { - return nil, err - } - - err = ipc.WaitAllGuestsConnections(guestConnections, socket) - if err != nil { - return nil, err - } - - return &RunnableGateway{ - Client: cl, - PodName: podName, - GatewayName: gatewayName, - Namespace: namespace, - Socket: socket, - GuestConnections: guestConnections, - }, nil -} - -// Start starts the ConcurrentRunnable. -func (rg *RunnableGateway) Start(ctx context.Context) error { - defer rg.Close() - - pods, err := ListAllGatewaysReplicas(ctx, rg.Client, rg.Namespace, rg.GatewayName) - if err != nil { - return err - } - - var activePod *corev1.Pod - - for i := range pods { - if pods[i].GetName() == rg.PodName { - activePod = &pods[i] - } else { - if err := RemoveActiveGatewayLabel(ctx, rg.Client, client.ObjectKeyFromObject(&pods[i])); err != nil { - return err - } - } - } - - if activePod == nil { - return fmt.Errorf("active gateway pod not found") - } - - if err := AddActiveGatewayLabel(ctx, rg.Client, client.ObjectKeyFromObject(activePod)); err != nil { - return err - } - - if err := ipc.StartAllGuestsConnections(rg.GuestConnections); err != nil { - return err - } - - return nil -} - -// Close closes the Runnable. -func (rg *RunnableGateway) Close() { - ipc.CloseListenSocket(rg.Socket) - ipc.CloseAllGuestsConnections(rg.GuestConnections) -} diff --git a/pkg/gateway/concurrent/guest.go b/pkg/gateway/concurrent/guest.go deleted file mode 100644 index 64b05ed265..0000000000 --- a/pkg/gateway/concurrent/guest.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2019-2026 The Liqo Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package concurrent - -import ( - "context" - "fmt" - "net" - "os" - - "sigs.k8s.io/controller-runtime/pkg/manager" - - "github.com/liqotech/liqo/pkg/utils/ipc" -) - -// Copyright 2019-2026 The Liqo Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -var _ manager.Runnable = &RunnableGuest{} - -// RunnableGuest is a RunnableGuest that manages concurrency. -type RunnableGuest struct { - GuestID string - Connection net.Conn -} - -// NewRunnableGuest creates a new Runnable. -func NewRunnableGuest(guestID string) (*RunnableGuest, error) { - conn, err := ipc.Connect(unixSocketPath, guestID) - if err != nil { - return nil, err - } - - return &RunnableGuest{ - GuestID: guestID, - Connection: conn, - }, nil -} - -// Start starts the ConcurrentRunnable. -func (rg *RunnableGuest) Start(_ context.Context) error { - if err := ipc.WaitForStart(rg.GuestID, rg.Connection); err != nil { - return err - } - return nil -} - -// Close closes the Runnable. -func (rg *RunnableGuest) Close() { - if err := rg.Connection.Close(); err != nil { - fmt.Printf("Error closing connection: %v\n", err) - os.Exit(1) - } -} diff --git a/pkg/gateway/concurrent/k8s.go b/pkg/gateway/concurrent/k8s.go deleted file mode 100644 index 95787ad9b0..0000000000 --- a/pkg/gateway/concurrent/k8s.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2019-2026 The Liqo Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package concurrent - -import ( - "context" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/liqotech/liqo/pkg/consts" -) - -// AddActiveGatewayLabel adds the active gateway label to the pod. -func AddActiveGatewayLabel(ctx context.Context, cl client.Client, key client.ObjectKey) error { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: key.Name, - Namespace: key.Namespace, - }, - } - if err := cl.Get(ctx, key, pod); err != nil { - return err - } - - labels := pod.GetLabels() - if labels == nil { - labels = map[string]string{} - } - labels[ActiveGatewayKey] = ActiveGatewayValue - pod.SetLabels(labels) - - if err := cl.Update(ctx, pod); err != nil { - return err - } - klog.Infof("Pod %s/%s is now the active gateway", pod.Namespace, pod.Name) - return nil -} - -// RemoveActiveGatewayLabel removes the active gateway label from the pod. -func RemoveActiveGatewayLabel(ctx context.Context, cl client.Client, key client.ObjectKey) error { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: key.Name, - Namespace: key.Namespace, - }, - } - if err := cl.Get(ctx, key, pod); err != nil { - return err - } - - labels := pod.GetLabels() - if labels == nil { - return nil - } - delete(labels, ActiveGatewayKey) - pod.SetLabels(labels) - - if err := cl.Update(ctx, pod); err != nil { - return err - } - klog.Infof("Pod %s/%s is no longer the active gateway", pod.Namespace, pod.Name) - return nil -} - -// ListAllGatewaysReplicas returns the list of all the gateways replicas of the same gateway. -func ListAllGatewaysReplicas(ctx context.Context, cl client.Client, namespace, gatewayName string) ([]corev1.Pod, error) { - podList := &corev1.PodList{} - if err := cl.List(ctx, podList, client.InNamespace(namespace), client.MatchingLabels{ - consts.K8sAppNameKey: gatewayName, - }); err != nil { - return nil, err - } - return podList.Items, nil -} diff --git a/pkg/gateway/concurrent/label.go b/pkg/gateway/concurrent/label.go deleted file mode 100644 index 6596446505..0000000000 --- a/pkg/gateway/concurrent/label.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2019-2026 The Liqo Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package concurrent - -const ( - // ActiveGatewayKey is the key used to label the active pod gateway. - ActiveGatewayKey = "networking.liqo.io/active" - // ActiveGatewayValue is the value used to label the active pod gateway. - ActiveGatewayValue = "true" -) diff --git a/pkg/gateway/connection/connections_controller.go b/pkg/gateway/connection/connections_controller.go index 98a0f262ac..c03cb5fbf6 100644 --- a/pkg/gateway/connection/connections_controller.go +++ b/pkg/gateway/connection/connections_controller.go @@ -39,6 +39,7 @@ import ( networkingv1beta1 "github.com/liqotech/liqo/apis/networking/v1beta1" "github.com/liqotech/liqo/pkg/conncheck" "github.com/liqotech/liqo/pkg/consts" + "github.com/liqotech/liqo/pkg/gateway/forge" "github.com/liqotech/liqo/pkg/gateway/tunnel" ) @@ -105,7 +106,7 @@ func (r *ConnectionsReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - status, err := r.ConnChecker.GetStatus(r.Options.GwOptions.RemoteClusterID) + status, err := r.ConnChecker.GetStatus(r.Options.GwOptions.Name) var ( latency time.Duration connStatusValue = networkingv1beta1.ConnectionError @@ -136,11 +137,12 @@ func (r *ConnectionsReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } -// ensureSender adds the conncheck sender for the reconciler's remote cluster if it isn't +// ensureSender adds the conncheck sender for this gateway if it isn't // already running. It is a no-op after the first successful call. func (r *ConnectionsReconciler) ensureSender(ctx context.Context, key types.NamespacedName) error { clusterID := r.Options.GwOptions.RemoteClusterID - if r.ConnChecker.HasSender(clusterID) { + gwName := r.Options.GwOptions.Name + if r.ConnChecker.HasSender(gwName) { return nil } @@ -149,8 +151,8 @@ func (r *ConnectionsReconciler) ensureSender(ctx context.Context, key types.Name return fmt.Errorf("unable to get the remote interface IP: %w", err) } - observer := onTransition(ObserveLatency(clusterID), r.enqueueTransition(key)) - if err := r.ConnChecker.AddSender(ctx, clusterID, remoteIP, observer); err != nil { + observer := onTransition(ObserveLatency(clusterID, gwName), r.enqueueTransition(key)) + if err := r.ConnChecker.AddSender(ctx, gwName, remoteIP, observer); err != nil { var dupErr *conncheck.DuplicateError if !errors.As(err, &dupErr) { return fmt.Errorf("unable to add the sender: %w", err) @@ -159,7 +161,7 @@ func (r *ConnectionsReconciler) ensureSender(ctx context.Context, key types.Name return nil } - go r.ConnChecker.RunSender(clusterID) + go r.ConnChecker.RunSender(gwName) return nil } @@ -197,8 +199,16 @@ func (r *ConnectionsReconciler) SetupWithManager(mgr ctrl.Manager) error { if err != nil { return err } + + // Each gateway pod owns a distinct Connection resource named gw-. + // Filter by name so each pod only reconciles its own Connection. + ownConnectionName := forge.GatewayResourceName(r.Options.GwOptions.Name) + filterByNamePredicate := predicate.NewPredicateFuncs(func(object client.Object) bool { + return object.GetName() == ownConnectionName + }) + return ctrl.NewControllerManagedBy(mgr).Named(consts.CtrlConnection). - For(&networkingv1beta1.Connection{}, builder.WithPredicates(filterByLabelsPredicate)). + For(&networkingv1beta1.Connection{}, builder.WithPredicates(filterByLabelsPredicate, filterByNamePredicate)). WatchesRawSource(source.Channel(r.transitions, &handler.EnqueueRequestForObject{})). Complete(r) } diff --git a/pkg/gateway/connection/metrics.go b/pkg/gateway/connection/metrics.go index f972a86696..c654fd9545 100644 --- a/pkg/gateway/connection/metrics.go +++ b/pkg/gateway/connection/metrics.go @@ -21,16 +21,15 @@ import ( "github.com/liqotech/liqo/pkg/gateway/tunnel" ) -const ( - driverLabel = "gateway" -) +const driverLabelValue = "wireguard" // ObserveLatency returns a conncheck.PingObserver that records the round-trip latency // into the prometheus metrics at the point of measurement (on each PONG). -func ObserveLatency(remoteClusterID string) conncheck.PingObserver { +func ObserveLatency(remoteClusterID, gatewayName string) conncheck.PingObserver { return tunnel.ObserveLatencyMetrics(tunnel.MetricsPeerLatency, tunnel.MetricsPeerLatencyHistogram, tunnel.MetricsPeerIsConnected, prometheus.Labels{ - tunnel.MetricsLabels[0]: driverLabel, + tunnel.MetricsLabels[0]: driverLabelValue, tunnel.MetricsLabels[1]: remoteClusterID, + tunnel.MetricsLabels[2]: gatewayName, }) } diff --git a/pkg/gateway/fabric/genevetunnel_controller.go b/pkg/gateway/fabric/genevetunnel_controller.go index b99b683e20..46b283cc6f 100644 --- a/pkg/gateway/fabric/genevetunnel_controller.go +++ b/pkg/gateway/fabric/genevetunnel_controller.go @@ -206,20 +206,30 @@ func (r *GeneveTunnelReconciler) SetupWithManager(mgr ctrl.Manager) error { return obj.GetNamespace() == r.Options.GwOptions.Namespace }) + // Each gateway pod should only reconcile GeneveTunnels belonging to its own InternalFabric. + gatewayPredicate := predicate.NewPredicateFuncs(func(obj client.Object) bool { + return obj.GetLabels()[consts.InternalFabricName] == r.Options.GwOptions.Name + }) + + internalFabricPredicate := predicate.NewPredicateFuncs(func(obj client.Object) bool { + return obj.GetName() == r.Options.GwOptions.Name + }) + return ctrl.NewControllerManagedBy(mgr).Named(consts.CtrlInternalNodeGeneve). For(&networkingv1beta1.GeneveTunnel{}, - builder.WithPredicates(namespacePredicate, predicate.GenerationChangedPredicate{})). + builder.WithPredicates(namespacePredicate, gatewayPredicate, predicate.GenerationChangedPredicate{})). Watches(&networkingv1beta1.InternalNode{}, handler.EnqueueRequestsFromMapFunc(r.internalNodeEnqueuer)). Watches(&networkingv1beta1.InternalFabric{}, handler.EnqueueRequestsFromMapFunc(r.internalFabricEnqueuer), - builder.WithPredicates(namespacePredicate)). + builder.WithPredicates(namespacePredicate, internalFabricPredicate)). Complete(r) } func (r *GeneveTunnelReconciler) internalNodeEnqueuer(ctx context.Context, obj client.Object) []reconcile.Request { list, err := getters.ListGeneveTunnelsByLabels(ctx, r.Client, r.Options.GwOptions.Namespace, labels.SelectorFromSet(labels.Set{ - consts.InternalNodeName: obj.GetName(), + consts.InternalNodeName: obj.GetName(), + consts.InternalFabricName: r.Options.GwOptions.Name, })) if err != nil { klog.Errorf("Failed to list genevetunnels for internalnode %s: %v", obj.GetName(), err) diff --git a/pkg/gateway/flags.go b/pkg/gateway/flags.go index 141028017e..d60ca1a224 100644 --- a/pkg/gateway/flags.go +++ b/pkg/gateway/flags.go @@ -39,8 +39,6 @@ const ( FlagNameNodeName FlagName = "node-name" // FlagNamePodName is the name of the pod. FlagNamePodName FlagName = "pod-name" - // FlagContainerName is the name of the container. - FlagContainerName FlagName = "container-name" // FlagNameGatewayUID is the UID of the Gateway resource. FlagNameGatewayUID FlagName = "gateway-uid" @@ -48,21 +46,9 @@ const ( // FlagNameMode is the mode in which the gateway is configured. FlagNameMode FlagName = "mode" - // FlagConcurrentContainersNames is the names of the containers that the gateway container must wait for. - FlagConcurrentContainersNames FlagName = "concurrent-containers-names" - // FlagNameReconcileTimeout is the reconciliation timeout. FlagNameReconcileTimeout FlagName = "reconcile-timeout" - // FlagNameLeaderElection is the flag to enable leader election. - FlagNameLeaderElection FlagName = "leader-election" - // FlagNameLeaderElectionLeaseDuration is the lease duration for the leader election. - FlagNameLeaderElectionLeaseDuration FlagName = "leader-election-lease-duration" - // FlagNameLeaderElectionRenewDeadline is the renew deadline for the leader election. - FlagNameLeaderElectionRenewDeadline FlagName = "leader-election-renew-deadline" - // FlagNameLeaderElectionRetryPeriod is the retry period for the leader election. - FlagNameLeaderElectionRetryPeriod FlagName = "leader-election-retry-period" - // FlagNameMetricsAddress is the address for the metrics endpoint. FlagNameMetricsAddress FlagName = "metrics-address" // FlagNameProbeAddr is the address for the health probe endpoint. @@ -100,7 +86,6 @@ var RequiredFlags = []FlagName{ FlagNameGatewayUID, FlagNameNodeName, FlagNamePodName, - FlagContainerName, } // InitFlags initializes the flags for the gateway. @@ -110,25 +95,13 @@ func InitFlags(flagset *pflag.FlagSet, opts *Options) { flagset.StringVar(&opts.RemoteClusterID, FlagNameRemoteClusterID.String(), "", "ClusterID of the remote cluster") flagset.StringVar(&opts.NodeName, FlagNameNodeName.String(), "", "Node name") flagset.StringVar(&opts.PodName, FlagNamePodName.String(), "", "Pod name") - flagset.StringVar(&opts.ContainerName, FlagContainerName.String(), "", "Container name") flagset.StringVar(&opts.GatewayUID, FlagNameGatewayUID.String(), "", "Parent gateway resource UID") flagset.Var(&opts.Mode, FlagNameMode.String(), "Parent gateway mode") - flagset.StringSliceVar(&opts.ConcurrentContainersNames, FlagConcurrentContainersNames.String(), - []string{}, "the container list that gateway container must wait for") - flagset.DurationVar(&opts.ReconcileTimeout, FlagNameReconcileTimeout.String(), 10*time.Second, "Reconciliation timeout") - flagset.BoolVar(&opts.LeaderElection, FlagNameLeaderElection.String(), false, "Enable leader election") - flagset.DurationVar(&opts.LeaderElectionLeaseDuration, FlagNameLeaderElectionLeaseDuration.String(), 15*time.Second, - "LeaseDuration for the leader election") - flagset.DurationVar(&opts.LeaderElectionRenewDeadline, FlagNameLeaderElectionRenewDeadline.String(), 10*time.Second, - "RenewDeadline for the leader election") - flagset.DurationVar(&opts.LeaderElectionRetryPeriod, FlagNameLeaderElectionRetryPeriod.String(), 2*time.Second, - "RetryPeriod for the leader election") - flagset.StringVar(&opts.MetricsAddress, FlagNameMetricsAddress.String(), "0", "Address for the metrics endpoint") flagset.StringVar(&opts.ProbeAddr, FlagNameProbeAddr.String(), "0", "Address for the health probe endpoint") flagset.StringVar(&opts.PprofAddr, FlagNamePprofAddr.String(), "", "Address for the pprof endpoint. Empty disables it") @@ -150,11 +123,5 @@ func MarkFlagsRequired(cmd *cobra.Command) error { } } - if cmd.Name() == "liqo-gateway" { - if err := cmd.MarkFlagRequired(FlagConcurrentContainersNames.String()); err != nil { - return err - } - } - return nil } diff --git a/pkg/gateway/label.go b/pkg/gateway/label.go index e329c59f41..dfd971d66b 100644 --- a/pkg/gateway/label.go +++ b/pkg/gateway/label.go @@ -17,7 +17,6 @@ package gateway import ( "github.com/liqotech/liqo/pkg/consts" "github.com/liqotech/liqo/pkg/firewall" - "github.com/liqotech/liqo/pkg/gateway/concurrent" "github.com/liqotech/liqo/pkg/route" ) @@ -50,8 +49,7 @@ const ( // ForgeActiveGatewayPodLabels returns the labels for the gateway pod. func ForgeActiveGatewayPodLabels() map[string]string { return map[string]string{ - concurrent.ActiveGatewayKey: concurrent.ActiveGatewayValue, - GatewayComponentKey: GatewayComponentGateway, + GatewayComponentKey: GatewayComponentGateway, } } diff --git a/pkg/gateway/options.go b/pkg/gateway/options.go index 0e8bd4b4ea..764e355b37 100644 --- a/pkg/gateway/options.go +++ b/pkg/gateway/options.go @@ -28,21 +28,13 @@ type Options struct { RemoteClusterID string NodeName string PodName string - ContainerName string GatewayUID string Mode Mode - ConcurrentContainersNames []string - ReconcileTimeout time.Duration - LeaderElection bool - LeaderElectionLeaseDuration time.Duration - LeaderElectionRenewDeadline time.Duration - LeaderElectionRetryPeriod time.Duration - MetricsAddress string ProbeAddr string PprofAddr string diff --git a/pkg/gateway/tunnel/metrics.go b/pkg/gateway/tunnel/metrics.go index 8132496ef1..95ab578d46 100644 --- a/pkg/gateway/tunnel/metrics.go +++ b/pkg/gateway/tunnel/metrics.go @@ -54,7 +54,7 @@ var ( // InitDefaultMetrics initializes the default metrics. func init() { - MetricsLabels = []string{"driver", "cluster_id"} + MetricsLabels = []string{"driver", "cluster_id", "gateway"} MetricsPeerReceivedBytes = prometheus.NewDesc( "liqo_peer_receive_bytes_total", diff --git a/pkg/gateway/tunnel/wireguard/metrics.go b/pkg/gateway/tunnel/wireguard/metrics.go index e3db25b5e1..9143c8ffc2 100644 --- a/pkg/gateway/tunnel/wireguard/metrics.go +++ b/pkg/gateway/tunnel/wireguard/metrics.go @@ -49,6 +49,7 @@ type PrometheusCollector struct { // MetricsOptions contains the options for the PrometheusCollector. type MetricsOptions struct { RemoteClusterID string + GatewayName string Namespace string WgImplementation WgImplementation } @@ -96,7 +97,7 @@ func (pc *PrometheusCollector) Collect(ch chan<- prometheus.Metric) { peer := device.Peers[0] - labels := []string{driverLabelValue, pc.metricsOptions.RemoteClusterID} + labels := []string{driverLabelValue, pc.metricsOptions.RemoteClusterID, pc.metricsOptions.GatewayName} ch <- prometheus.MustNewConstMetric( tunnel.MetricsPeerReceivedBytes, diff --git a/pkg/liqo-controller-manager/core/foreigncluster-controller/status.go b/pkg/liqo-controller-manager/core/foreigncluster-controller/status.go index 91d1b3914f..17b600b7af 100644 --- a/pkg/liqo-controller-manager/core/foreigncluster-controller/status.go +++ b/pkg/liqo-controller-manager/core/foreigncluster-controller/status.go @@ -21,6 +21,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" @@ -28,6 +29,7 @@ import ( liqov1beta1 "github.com/liqotech/liqo/apis/core/v1beta1" networkingv1beta1 "github.com/liqotech/liqo/apis/networking/v1beta1" offloadingv1beta1 "github.com/liqotech/liqo/apis/offloading/v1beta1" + "github.com/liqotech/liqo/pkg/consts" "github.com/liqotech/liqo/pkg/gateway/forge" "github.com/liqotech/liqo/pkg/utils" fcutils "github.com/liqotech/liqo/pkg/utils/foreigncluster" @@ -66,9 +68,14 @@ func (r *ForeignClusterReconciler) handleConnectionStatus(ctx context.Context, fc *liqov1beta1.ForeignCluster, statusExceptions map[liqov1beta1.ConditionType]statusException) error { clusterID := fc.Spec.ClusterID - connection, err := getters.GetConnectionByClusterID(ctx, r.Client, string(clusterID)) - switch { - case errors.IsNotFound(err): + remoteClusterIDSelector := labels.Set{consts.RemoteClusterID: string(clusterID)}.AsSelector() + connections, err := getters.ListConnectionsByLabel(ctx, r.Client, corev1.NamespaceAll, remoteClusterIDSelector) + if err != nil { + klog.Errorf("an error occurred while listing the Connection resources for the ForeignCluster %q: %s", clusterID, err) + return err + } + + if len(connections.Items) == 0 { klog.V(6).Infof("Connection resource not found for ForeignCluster %q", clusterID) fcutils.DeleteModuleCondition(&fc.Status.Modules.Networking, liqov1beta1.NetworkConnectionStatusCondition) statusExceptions[liqov1beta1.NetworkConnectionStatusCondition] = statusException{ @@ -76,26 +83,39 @@ func (r *ForeignClusterReconciler) handleConnectionStatus(ctx context.Context, Reason: connectionMissingReason, Message: connectionMissingMessage, } - case err != nil: - klog.Errorf("an error occurred while getting the Connection resource for the ForeignCluster %q: %s", clusterID, err) - return err - default: - fcutils.EnableModuleNetworking(fc) - switch connection.Status.Value { + return nil + } + + fcutils.EnableModuleNetworking(fc) + + // With multiple gateways, check if at least one connection is Connected. + anyConnected := false + anyConnecting := false + for i := range connections.Items { + switch connections.Items[i].Status.Value { case networkingv1beta1.Connected: - fcutils.EnsureModuleCondition(&fc.Status.Modules.Networking, - liqov1beta1.NetworkConnectionStatusCondition, liqov1beta1.ConditionStatusEstablished, - connectionEstablishedReason, connectionEstablishedMessage) + anyConnected = true case networkingv1beta1.Connecting: - fcutils.EnsureModuleCondition(&fc.Status.Modules.Networking, - liqov1beta1.NetworkConnectionStatusCondition, liqov1beta1.ConditionStatusPending, - connectionPendingReason, connectionPendingMessage) + anyConnecting = true default: - fcutils.EnsureModuleCondition(&fc.Status.Modules.Networking, - liqov1beta1.NetworkConnectionStatusCondition, liqov1beta1.ConditionStatusError, - connectionErrorReason, connectionErrorMessage) + // ConnectionError or empty — nothing to track. } } + + switch { + case anyConnected: + fcutils.EnsureModuleCondition(&fc.Status.Modules.Networking, + liqov1beta1.NetworkConnectionStatusCondition, liqov1beta1.ConditionStatusEstablished, + connectionEstablishedReason, connectionEstablishedMessage) + case anyConnecting: + fcutils.EnsureModuleCondition(&fc.Status.Modules.Networking, + liqov1beta1.NetworkConnectionStatusCondition, liqov1beta1.ConditionStatusPending, + connectionPendingReason, connectionPendingMessage) + default: + fcutils.EnsureModuleCondition(&fc.Status.Modules.Networking, + liqov1beta1.NetworkConnectionStatusCondition, liqov1beta1.ConditionStatusError, + connectionErrorReason, connectionErrorMessage) + } return nil } @@ -103,10 +123,19 @@ func (r *ForeignClusterReconciler) handleGatewaysStatus(ctx context.Context, fc *liqov1beta1.ForeignCluster, statusExceptions map[liqov1beta1.ConditionType]statusException) error { clusterID := fc.Spec.ClusterID - gwServer, errServer := getters.GetGatewayServerByClusterID(ctx, r.Client, clusterID, corev1.NamespaceAll) - gwClient, errClient := getters.GetGatewayClientByClusterID(ctx, r.Client, clusterID, corev1.NamespaceAll) + gwServerList, errServer := getters.ListGatewayServersByClusterID(ctx, r.Client, clusterID, corev1.NamespaceAll) + gwClientList, errClient := getters.ListGatewayClientsByClusterID(ctx, r.Client, clusterID, corev1.NamespaceAll) - if errors.IsNotFound(errServer) && errors.IsNotFound(errClient) { + serverCount := 0 + if errServer == nil && gwServerList != nil { + serverCount = len(gwServerList.Items) + } + clientCount := 0 + if errClient == nil && gwClientList != nil { + clientCount = len(gwClientList.Items) + } + + if serverCount == 0 && clientCount == 0 { klog.V(6).Infof("Both GatewayServer and GatewayClient resources not found for ForeignCluster %q", clusterID) statusExceptions[liqov1beta1.NetworkGatewayPresenceCondition] = statusException{ ConditionStatusType: liqov1beta1.ConditionStatusNotReady, @@ -121,31 +150,19 @@ func (r *ForeignClusterReconciler) handleGatewaysStatus(ctx context.Context, } } - switch { - case errors.IsNotFound(errServer): + // Handle GatewayServer status + if serverCount == 0 { klog.V(6).Infof("GatewayServer resource not found for ForeignCluster %q", clusterID) fcutils.DeleteModuleCondition(&fc.Status.Modules.Networking, liqov1beta1.NetworkGatewayServerStatusCondition) - case errServer != nil: - klog.Errorf("an error occurred while getting the GatewayServer resource for the ForeignCluster %q: %s", clusterID, errServer) - return errServer - default: + } else { fcutils.EnableModuleNetworking(fc) - gwDeployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: forge.GatewayResourceName(gwServer.GetName()), - Namespace: gwServer.GetNamespace(), - }, - } - if err := r.Get(ctx, client.ObjectKeyFromObject(gwDeployment), gwDeployment); err != nil { - klog.Errorf("an error occurred while getting the GatewayServer deployment for the ForeignCluster %q: %s", clusterID, err) - return err - } + allReady, someNotReady := r.checkGwServerDeploymentsReady(ctx, gwServerList.Items) switch { - case gwDeployment.Status.ReadyReplicas == 0: + case !allReady && !someNotReady: fcutils.EnsureModuleCondition(&fc.Status.Modules.Networking, liqov1beta1.NetworkGatewayServerStatusCondition, liqov1beta1.ConditionStatusNotReady, gatewaysNotReadyReason, gatewaysNotReadyMessage) - case gwDeployment.Status.UnavailableReplicas > 0: + case someNotReady: fcutils.EnsureModuleCondition(&fc.Status.Modules.Networking, liqov1beta1.NetworkGatewayServerStatusCondition, liqov1beta1.ConditionStatusSomeNotReady, gatewaySomeNotReadyReason, gatewaySomeNotReadyMessage) @@ -156,31 +173,19 @@ func (r *ForeignClusterReconciler) handleGatewaysStatus(ctx context.Context, } } - switch { - case errors.IsNotFound(errClient): + // Handle GatewayClient status + if clientCount == 0 { klog.V(6).Infof("GatewayClient resource not found for ForeignCluster %q", clusterID) fcutils.DeleteModuleCondition(&fc.Status.Modules.Networking, liqov1beta1.NetworkGatewayClientStatusCondition) - case errClient != nil: - klog.Errorf("an error occurred while getting the GatewayClient resource for the ForeignCluster %q: %s", clusterID, errClient) - return errClient - default: + } else { fcutils.EnableModuleNetworking(fc) - gwDeployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: forge.GatewayResourceName(gwClient.GetName()), - Namespace: gwClient.GetNamespace(), - }, - } - if err := r.Get(ctx, client.ObjectKeyFromObject(gwDeployment), gwDeployment); err != nil { - klog.Errorf("an error occurred while getting the GatewayClient deployment for the ForeignCluster %q: %s", clusterID, err) - return err - } + allReady, someNotReady := r.checkGwClientDeploymentsReady(ctx, gwClientList.Items) switch { - case gwDeployment.Status.ReadyReplicas == 0: + case !allReady && !someNotReady: fcutils.EnsureModuleCondition(&fc.Status.Modules.Networking, liqov1beta1.NetworkGatewayClientStatusCondition, liqov1beta1.ConditionStatusNotReady, gatewaysNotReadyReason, gatewaysNotReadyMessage) - case gwDeployment.Status.UnavailableReplicas > 0: + case someNotReady: fcutils.EnsureModuleCondition(&fc.Status.Modules.Networking, liqov1beta1.NetworkGatewayClientStatusCondition, liqov1beta1.ConditionStatusSomeNotReady, gatewaySomeNotReadyReason, gatewaySomeNotReadyMessage) @@ -194,6 +199,63 @@ func (r *ForeignClusterReconciler) handleGatewaysStatus(ctx context.Context, return nil } +func (r *ForeignClusterReconciler) checkGwClientDeploymentsReady(ctx context.Context, + gwClientList []networkingv1beta1.GatewayClient) (allReady, someNotReady bool) { + allReady = true + anyReady := false + for i := range gwClientList { + gw := &gwClientList[i] + ready, err := r.isGatewayDeploymentReady(ctx, gw.GetName(), gw.GetNamespace()) + if err != nil { + klog.Errorf("getting the GatewayClient deployment %q: %s", gw.GetName(), err) + allReady = false + continue + } + + if ready { + anyReady = true + } else { + allReady = false + } + } + return allReady, !allReady && anyReady +} + +func (r *ForeignClusterReconciler) checkGwServerDeploymentsReady(ctx context.Context, + gwServerList []networkingv1beta1.GatewayServer) (allReady, someNotReady bool) { + allReady = true + anyReady := false + for i := range gwServerList { + gw := &gwServerList[i] + ready, err := r.isGatewayDeploymentReady(ctx, gw.GetName(), gw.GetNamespace()) + if err != nil { + klog.Errorf("getting the GatewayServer deployment %q: %s", gw.GetName(), err) + allReady = false + continue + } + + if ready { + anyReady = true + } else { + allReady = false + } + } + return allReady, !allReady && anyReady +} + +func (r *ForeignClusterReconciler) isGatewayDeploymentReady(ctx context.Context, gatewayName, gatewayNamespace string) (bool, error) { + gwDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: forge.GatewayResourceName(gatewayName), + Namespace: gatewayNamespace, + }, + } + if err := r.Get(ctx, client.ObjectKeyFromObject(gwDeployment), gwDeployment); err != nil { + return false, err + } + return gwDeployment.Status.ReadyReplicas > 0 && gwDeployment.Status.UnavailableReplicas == 0, nil +} + func (r *ForeignClusterReconciler) handleNetworkConfigurationStatus(ctx context.Context, fc *liqov1beta1.ForeignCluster, statusExceptions map[liqov1beta1.ConditionType]statusException) error { clusterID := fc.Spec.ClusterID diff --git a/pkg/liqo-controller-manager/ipmapping/configuration_controller.go b/pkg/liqo-controller-manager/ipmapping/configuration_controller.go deleted file mode 100644 index d6a528ade1..0000000000 --- a/pkg/liqo-controller-manager/ipmapping/configuration_controller.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2019-2026 The Liqo Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ipmapping - -import ( - "context" - "fmt" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/tools/record" - "k8s.io/klog/v2" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - - ipamv1alpha1 "github.com/liqotech/liqo/apis/ipam/v1alpha1" - networkingv1beta1 "github.com/liqotech/liqo/apis/networking/v1beta1" - "github.com/liqotech/liqo/pkg/consts" - networkingutils "github.com/liqotech/liqo/pkg/liqo-controller-manager/networking/utils" - cidrutils "github.com/liqotech/liqo/pkg/utils/cidr" - ipamutils "github.com/liqotech/liqo/pkg/utils/ipam" - "github.com/liqotech/liqo/pkg/utils/resource" -) - -// cluster-role -// +kubebuilder:rbac:groups=networking.liqo.io,resources=configurations,verbs=get;list;watch -// +kubebuilder:rbac:groups=ipam.liqo.io,resources=ips,verbs=get;list;watch;create;update;patch;delete - -// ConfigurationReconciler creates a mapping for the UnknownSourceIP for each remote cluster. -// This allows traffic with "external" source IPs to be routed from a leaf cluster to another. -type ConfigurationReconciler struct { - Client client.Client - Scheme *runtime.Scheme - EventsRecorder record.EventRecorder -} - -func forgeUnknownSourceIPName(cfg *networkingv1beta1.Configuration) string { - return cfg.Name + "-unknown-source" -} - -// NewConfigurationReconciler returns a new ConfigurationReconciler. -func NewConfigurationReconciler(cl client.Client, s *runtime.Scheme, er record.EventRecorder) *ConfigurationReconciler { - return &ConfigurationReconciler{ - Client: cl, - Scheme: s, - EventsRecorder: er, - } -} - -// Reconcile manage Configuration resources. -func (r *ConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - cfg := &networkingv1beta1.Configuration{} - if err := r.Client.Get(ctx, req.NamespacedName, cfg); err != nil { - if apierrors.IsNotFound(err) { - klog.V(6).Infof("There is no configuration %s", req.String()) - return ctrl.Result{}, nil - } - return ctrl.Result{}, fmt.Errorf("unable to get the configuration %q: %w", req.NamespacedName, err) - } - klog.V(4).Infof("Reconciling configuration %q", req.NamespacedName) - - if len(cfg.Status.Remote.CIDR.External) == 0 { - return ctrl.Result{}, fmt.Errorf("configuration %q has no remote external CIDR", req.NamespacedName) - } - remoteExternalCIDR := cidrutils.GetPrimary(cfg.Status.Remote.CIDR.External).String() - remoteUnknownSourceIP, err := ipamutils.GetUnknownSourceIP(remoteExternalCIDR) - if err != nil { - return ctrl.Result{}, fmt.Errorf("unable to get the unknown source IP: %w", err) - } - - if err := r.deletePreviousUnknownSourceIPs(ctx, cfg, remoteUnknownSourceIP); err != nil { - return ctrl.Result{}, fmt.Errorf("unable to delete orphan unknown source IPs: %w", err) - } - - if err := r.createOrUpdateUnknownSourceIPResource(ctx, cfg, remoteUnknownSourceIP); err != nil { - return ctrl.Result{}, fmt.Errorf("unable to create or update the unknown source IP: %w", err) - } - - return ctrl.Result{}, nil -} - -// SetupWithManager register the RemappingReconciler to the manager. -func (r *ConfigurationReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr).Named(consts.CtrlConfigurationIPMapping). - For(&networkingv1beta1.Configuration{}, builder.WithPredicates(networkingutils.AreConfigurationNetworkCIDRsConfiguredPredicate())). - Complete(r) -} - -func (r *ConfigurationReconciler) createOrUpdateUnknownSourceIPResource(ctx context.Context, - cfg *networkingv1beta1.Configuration, remoteUnknownSourceIP string) error { - ip := &ipamv1alpha1.IP{ - ObjectMeta: metav1.ObjectMeta{ - Name: forgeUnknownSourceIPName(cfg), - Namespace: cfg.Namespace, - Labels: map[string]string{ - consts.RemoteClusterID: cfg.GetName(), - }, - }, - } - if _, err := resource.CreateOrUpdate(ctx, r.Client, ip, func() error { - ip.Spec = ipamv1alpha1.IPSpec{ - IP: networkingv1beta1.IP(remoteUnknownSourceIP), - } - return controllerutil.SetOwnerReference(cfg, ip, r.Scheme) - }); err != nil { - return fmt.Errorf("unable to create or update the IP %q: %w", ip.Name, err) - } - return nil -} - -// deletePreviousUnknownSourceIPs removes any IP resource owned by cfg with the same RemoteClusterID -// label whose IP doesn't match the desired one. -func (r *ConfigurationReconciler) deletePreviousUnknownSourceIPs(ctx context.Context, - cfg *networkingv1beta1.Configuration, remoteUnknownSourceIP string) error { - unknownSourceIPName := forgeUnknownSourceIPName(cfg) - - unknownSourceIP := &ipamv1alpha1.IP{} - err := r.Client.Get(ctx, client.ObjectKey{Namespace: cfg.Namespace, Name: unknownSourceIPName}, unknownSourceIP) - switch { - case apierrors.IsNotFound(err): - // No previous IP exists, nothing to delete. - return nil - case err != nil: - return fmt.Errorf("unable to get the IP %q: %w", unknownSourceIPName, err) - case unknownSourceIP.Spec.IP == networkingv1beta1.IP(remoteUnknownSourceIP): - // The current IP is correct, no need to delete any orphan. - return nil - } - - err = r.Client.Delete(ctx, unknownSourceIP) - if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("unable to delete orphan IP %q: %w", unknownSourceIP.Name, err) - } - - klog.Infof("Deleted previous Unknown source IP %q for configuration %q", unknownSourceIPName, cfg.Name) - return nil -} diff --git a/pkg/liqo-controller-manager/ipmapping/pod_controller.go b/pkg/liqo-controller-manager/ipmapping/pod_controller.go index b7335298d6..5e9e4f32f0 100644 --- a/pkg/liqo-controller-manager/ipmapping/pod_controller.go +++ b/pkg/liqo-controller-manager/ipmapping/pod_controller.go @@ -51,12 +51,13 @@ func NewOffloadedPodReconciler(cl client.Client, s *runtime.Scheme, er record.Ev } // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch // +kubebuilder:rbac:groups=networking.liqo.io,resources=ips,verbs=get;list;watch;update;patch;create;delete // Reconcile reconciles on offloaded pods. func (r *OffloadedPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { pod := &corev1.Pod{} - if err := r.Client.Get(ctx, req.NamespacedName, pod); err != nil { + if err := r.Get(ctx, req.NamespacedName, pod); err != nil { if apierrors.IsNotFound(err) { klog.V(6).Infof("There is no pod %s", req.String()) return ctrl.Result{}, nil @@ -70,6 +71,22 @@ func (r *OffloadedPodReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{RequeueAfter: time.Second}, nil } + if pod.Spec.NodeName == "" { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + + node := &corev1.Node{} + if err := r.Get(ctx, client.ObjectKey{Name: pod.Spec.NodeName}, node); err != nil { + return ctrl.Result{}, fmt.Errorf("fetching node %q for pod %s: %w", pod.Spec.NodeName, req.String(), err) + } + + // IP mappings are needed only for pods running on a remote cluster through a VirtualNode + // (i.e., a node acting as provider for the local cluster). + if nodeType, ok := node.Labels[consts.TypeLabel]; !ok || nodeType != consts.TypeNode { + klog.V(4).Infof("Pod %s is scheduled on real node %q, skipping IP mapping", req.String(), node.Name) + return ctrl.Result{}, nil + } + op, err := CreateOrUpdateIP(ctx, r.Client, r.Scheme, pod) if err != nil { return ctrl.Result{}, fmt.Errorf("creating or updating IP for pod %s: %w", req.String(), err) diff --git a/pkg/liqo-controller-manager/networking/external-network/client-operator/client_controller.go b/pkg/liqo-controller-manager/networking/external-network/client-operator/client_controller.go index 787026ed93..0502df561e 100644 --- a/pkg/liqo-controller-manager/networking/external-network/client-operator/client_controller.go +++ b/pkg/liqo-controller-manager/networking/external-network/client-operator/client_controller.go @@ -227,6 +227,17 @@ func (r *ClientReconciler) EnsureGatewayClient(ctx context.Context, gwClient *ne resource.AddGlobalAnnotations(objChild) + // Report the source template identity on the rendered resource so that peer gateway controllers can filter + // and compare template revisions. + annotations := objChild.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[consts.TemplateNameAnnotationKey] = template.GetName() + annotations[consts.TemplateNamespaceAnnotationKey] = template.GetNamespace() + annotations[consts.TemplateGenerationAnnotationKey] = fmt.Sprintf("%d", template.GetGeneration()) + objChild.SetAnnotations(annotations) + objChild.SetOwnerReferences([]metav1.OwnerReference{ { APIVersion: gwClient.APIVersion, diff --git a/pkg/liqo-controller-manager/networking/external-network/remapping/ip.go b/pkg/liqo-controller-manager/networking/external-network/remapping/ip.go index 5af2f065cd..02798674a9 100644 --- a/pkg/liqo-controller-manager/networking/external-network/remapping/ip.go +++ b/pkg/liqo-controller-manager/networking/external-network/remapping/ip.go @@ -193,8 +193,8 @@ func enforceFirewallConfigurationMasqSpec(fwcfg *networkingv1beta1.FirewallConfi } func enforceFirewallConfigurationChains(fwcfg *networkingv1beta1.FirewallConfiguration, ip *ipamv1alpha1.IP) { - if fwcfg.Spec.Table.Chains == nil || len(fwcfg.Spec.Table.Chains) != 2 { - fwcfg.Spec.Table.Chains = make([]firewall.Chain, 2) + if fwcfg.Spec.Table.Chains == nil || len(fwcfg.Spec.Table.Chains) != 1 { + fwcfg.Spec.Table.Chains = make([]firewall.Chain, 1) } chainPre := &fwcfg.Spec.Table.Chains[0] chainPre.Name = &PreroutingChainName @@ -203,14 +203,6 @@ func enforceFirewallConfigurationChains(fwcfg *networkingv1beta1.FirewallConfigu chainPre.Hook = &firewall.ChainHookPrerouting chainPre.Priority = ptr.To(firewall.ChainPriorityNATDest) ensureFirewallConfigurationDNATRules(&chainPre.Rules, ip) - - chainPost := &fwcfg.Spec.Table.Chains[1] - chainPost.Name = &PostroutingChainName - chainPost.Policy = ptr.To(firewall.ChainPolicyAccept) - chainPost.Type = firewall.ChainTypeNAT - chainPost.Hook = &firewall.ChainHookPostrouting - chainPost.Priority = ptr.To(firewall.ChainPriorityNATSource) - ensureFirewallConfigurationSNATRules(&chainPost.Rules, ip) } func enforceFirewallConfigurationMasqChains(fwcfg *networkingv1beta1.FirewallConfiguration, ip *ipamv1alpha1.IP) { @@ -267,26 +259,6 @@ func ensureFirewallConfigurationDNATRules(rules *firewall.RulesSet, ip *ipamv1al } } -func ensureFirewallConfigurationSNATRules(rules *firewall.RulesSet, ip *ipamv1alpha1.IP) { - remappedIP := ipamutils.GetRemappedIP(ip) - if !containsNATRule(rules.NatRules, ip.Spec.IP.String(), firewall.MatchPositionSrc) { - rules.NatRules = append(rules.NatRules, firewall.NatRule{ - NatType: firewall.NatTypeSource, - To: ptr.To(ip.Spec.IP.String()), - Name: &ip.Name, - Match: []firewall.Match{ - { - Op: firewall.MatchOperationEq, - IP: &firewall.MatchIP{ - Position: firewall.MatchPositionSrc, - Value: remappedIP.String(), - }, - }, - }, - }) - } -} - func containsNatRuleMasquerade(rules []firewall.NatRule, dst string) bool { for i := range rules { if rules[i].To == nil && rules[i].NatType == firewall.NatTypeMasquerade { diff --git a/pkg/liqo-controller-manager/networking/external-network/route/configuration_controller.go b/pkg/liqo-controller-manager/networking/external-network/route/configuration_controller.go index def926a46f..98deb731b2 100644 --- a/pkg/liqo-controller-manager/networking/external-network/route/configuration_controller.go +++ b/pkg/liqo-controller-manager/networking/external-network/route/configuration_controller.go @@ -56,7 +56,7 @@ func NewConfigurationReconciler(cl client.Client, s *runtime.Scheme, // cluster-role // +kubebuilder:rbac:groups=networking.liqo.io,resources=configurations,verbs=get;list;watch;update;patch // +kubebuilder:rbac:groups=networking.liqo.io,resources=routeconfigurations,verbs=get;list;watch;update;patch;create;delete -// +kubebuilder:rbac:groups=networking.liqo.io,resources=internalnodes,verbs=get;list;watch +// +kubebuilder:rbac:groups=networking.liqo.io,resources=firewallconfigurations,verbs=get;list;watch;update;patch;create;delete // +kubebuilder:rbac:groups=networking.liqo.io,resources=gatewayservers,verbs=get;list;watch // +kubebuilder:rbac:groups=networking.liqo.io,resources=gatewayclients,verbs=get;list;watch @@ -92,28 +92,9 @@ func (r *ConfigurationReconciler) SetupWithManager(mgr ctrl.Manager) error { &networkingv1beta1.GatewayClient{}, handler.EnqueueRequestsFromMapFunc(r.configurationEnqueuerByRemoteID()), ). - Watches( - &networkingv1beta1.InternalNode{}, - handler.EnqueueRequestsFromMapFunc(r.configurationEnqueuerForAllConfigurations()), - ). Complete(r) } -func (r *ConfigurationReconciler) configurationEnqueuerForAllConfigurations() handler.MapFunc { - return func(ctx context.Context, _ client.Object) []reconcile.Request { - var cfgList networkingv1beta1.ConfigurationList - if err := r.List(ctx, &cfgList); err != nil { - klog.Errorf("unable to list configurations: %s", err) - return nil - } - requests := make([]reconcile.Request, 0, len(cfgList.Items)) - for i := range cfgList.Items { - requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&cfgList.Items[i])}) - } - return requests - } -} - func (r *ConfigurationReconciler) configurationEnqueuerByRemoteID() handler.MapFunc { return func(ctx context.Context, obj client.Object) []reconcile.Request { labels := obj.GetLabels() diff --git a/pkg/liqo-controller-manager/networking/external-network/route/k8s.go b/pkg/liqo-controller-manager/networking/external-network/route/k8s.go index 046591d6d6..65c6310b30 100644 --- a/pkg/liqo-controller-manager/networking/external-network/route/k8s.go +++ b/pkg/liqo-controller-manager/networking/external-network/route/k8s.go @@ -20,7 +20,6 @@ import ( "slices" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -28,18 +27,41 @@ import ( liqov1beta1 "github.com/liqotech/liqo/apis/core/v1beta1" networkingv1beta1 "github.com/liqotech/liqo/apis/networking/v1beta1" + "github.com/liqotech/liqo/apis/networking/v1beta1/firewall" "github.com/liqotech/liqo/pkg/consts" "github.com/liqotech/liqo/pkg/gateway" "github.com/liqotech/liqo/pkg/gateway/tunnel" + "github.com/liqotech/liqo/pkg/liqo-controller-manager/networking/external-network/remapping" + internalnetwork "github.com/liqotech/liqo/pkg/liqo-controller-manager/networking/internal-network" "github.com/liqotech/liqo/pkg/utils/getters" "github.com/liqotech/liqo/pkg/utils/resource" ) +const ( + // gwExtMark is the fwmark value used to tag traffic arriving on Geneve interfaces (liqo.*). + // It allows the gw-ext RouteConfiguration to match on FwMark + Dst instead of Iif + Dst, + // collapsing N*R rules to R rules while still preventing routing loops (packets arriving + // on the WireGuard interface liqo-tunnel are not marked and do not match). + // The mark is set per-packet in the prerouting chain (not via conntrack) so it is available + // for route lookup and does not leak to return traffic. + // + // The value 0xFF00 is chosen to avoid collision with the internal-network mark allocator + // (pkg/liqo-controller-manager/networking/internal-network/route/mark.go), which assigns + // sequential marks starting from 1, one per node. A high value ensures no overlap even in + // very large clusters. + gwExtMark = 0xFF00 +) + // GenerateRouteConfigurationName generates the name of the RouteConfiguration object. func GenerateRouteConfigurationName(cfg *networkingv1beta1.Configuration) string { return fmt.Sprintf("%s-gw-ext", cfg.Name) } +// GenerateFirewallConfigurationName generates the name of the FirewallConfiguration object. +func GenerateFirewallConfigurationName(cfg *networkingv1beta1.Configuration) string { + return fmt.Sprintf("%s-gw-ext", cfg.Name) +} + // GetRemoteClusterID returns the remote cluster ID of the Configuration. func GetRemoteClusterID(cfg *networkingv1beta1.Configuration) (liqov1beta1.ClusterID, error) { if cfg.GetLabels() == nil { @@ -52,7 +74,8 @@ func GetRemoteClusterID(cfg *networkingv1beta1.Configuration) (liqov1beta1.Clust return liqov1beta1.ClusterID(remoteID), nil } -// enforceRouteConfigurationPresence creates or updates a RouteConfiguration object. +// enforceRouteConfigurationPresence creates or updates a RouteConfiguration object and its +// associated FirewallConfiguration. func enforceRouteConfigurationPresence(ctx context.Context, cl client.Client, scheme *runtime.Scheme, cfg *networkingv1beta1.Configuration) error { remoteClusterID, err := GetRemoteClusterID(cfg) @@ -74,28 +97,93 @@ func enforceRouteConfigurationPresence(ctx context.Context, cl client.Client, sc return err } - routecfg := &networkingv1beta1.RouteConfiguration{ + // Ensure the FirewallConfiguration that marks traffic arriving on Geneve interfaces. + fwcfg := &networkingv1beta1.FirewallConfiguration{ ObjectMeta: metav1.ObjectMeta{ - Name: GenerateRouteConfigurationName(cfg), + Name: GenerateFirewallConfigurationName(cfg), Namespace: cfg.Namespace, }, } - - internalNodes, err := getters.ListInternalNodesByLabels(ctx, cl, labels.Everything()) - if err != nil { - return err + if _, err = resource.CreateOrUpdate(ctx, cl, fwcfg, + forgeMutateFirewallConfiguration(cfg, fwcfg, scheme, remoteClusterID)); err != nil { + return fmt.Errorf("ensuring firewall configuration %q: %w", fwcfg.Name, err) } + routecfg := &networkingv1beta1.RouteConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: GenerateRouteConfigurationName(cfg), + Namespace: cfg.Namespace, + }, + } _, err = resource.CreateOrUpdate(ctx, cl, routecfg, - forgeMutateRouteConfiguration(cfg, routecfg, scheme, remoteClusterID, remoteInterfaceIP, internalNodes)) + forgeMutateRouteConfiguration(cfg, routecfg, scheme, remoteClusterID, remoteInterfaceIP)) return err } +// forgeMutateFirewallConfiguration mutates a FirewallConfiguration object that marks traffic +// arriving on any Geneve interface (liqo.*) with a constant fwmark. The mark is set directly +// on the packet (not via conntrack) in the prerouting chain at mangle priority, so it is +// available for ip rule route lookup. This is per-packet, not per-connection, preventing +// return traffic on liqo-tunnel from being marked. +func forgeMutateFirewallConfiguration(cfg *networkingv1beta1.Configuration, + fwcfg *networkingv1beta1.FirewallConfiguration, scheme *runtime.Scheme, + remoteClusterID liqov1beta1.ClusterID) func() error { + return func() error { + if err := controllerutil.SetOwnerReference(cfg, fwcfg, scheme); err != nil { + return err + } + + fwcfg.Labels = remapping.ForgeFirewallTargetLabels(string(remoteClusterID)) + + markValue := fmt.Sprintf("%d", gwExtMark) + + fwcfg.Spec = networkingv1beta1.FirewallConfigurationSpec{ + Table: firewall.Table{ + Name: ptr.To(cfg.Name), + Family: ptr.To(firewall.TableFamilyIPv4), + Chains: []firewall.Chain{ + { + // Prerouting chain at mangle priority: set the packet fwmark directly for any + // packet arriving on a liqo.* interface. Runs before route lookup so ip rule + // can match on FwMark. The WireGuard interface (liqo-tunnel) uses a dash, not + // a dot, so it does not match the "liqo." prefix. + Name: ptr.To("gw-ext-mark"), + Type: firewall.ChainTypeFilter, + Policy: ptr.To(firewall.ChainPolicyAccept), + Hook: ptr.To(firewall.ChainHookPrerouting), + Priority: &firewall.ChainPriorityMangle, + Rules: firewall.RulesSet{ + FilterRules: []firewall.FilterRule{ + { + Name: ptr.To("gw-ext-mark"), + Match: []firewall.Match{ + { + Op: firewall.MatchOperationEq, + Dev: &firewall.MatchDev{ + Value: internalnetwork.InterfaceNamePrefix, + Position: firewall.MatchDevPositionIn, + Wildcard: true, + }, + }, + }, + Action: firewall.ActionSetMetaMark, + Value: ptr.To(markValue), + }, + }, + }, + }, + }, + }, + } + return nil + } +} + // forgeMutateRouteConfiguration mutates a RouteConfiguration object. func forgeMutateRouteConfiguration(cfg *networkingv1beta1.Configuration, routecfg *networkingv1beta1.RouteConfiguration, scheme *runtime.Scheme, remoteClusterID liqov1beta1.ClusterID, - remoteInterfaceIP string, internalNodes *networkingv1beta1.InternalNodeList) func() error { + remoteInterfaceIP string) func() error { return func() error { var err error @@ -112,21 +200,19 @@ func forgeMutateRouteConfiguration(cfg *networkingv1beta1.Configuration, } remoteCIDRs := slices.Concat(cfg.Spec.Remote.CIDR.Pod, cfg.Spec.Remote.CIDR.External) - for i := range internalNodes.Items { - iif := &internalNodes.Items[i].Spec.Interface.Gateway.Name - for j := range remoteCIDRs { - dst := &remoteCIDRs[j] - routecfg.Spec.Table.Rules = append(routecfg.Spec.Table.Rules, networkingv1beta1.Rule{ - Iif: iif, - Dst: dst, - Routes: []networkingv1beta1.Route{ - { - Dst: dst, - Gw: ptr.To(networkingv1beta1.IP(remoteInterfaceIP)), - }, + mark := gwExtMark + for j := range remoteCIDRs { + dst := &remoteCIDRs[j] + routecfg.Spec.Table.Rules = append(routecfg.Spec.Table.Rules, networkingv1beta1.Rule{ + FwMark: &mark, + Dst: dst, + Routes: []networkingv1beta1.Route{ + { + Dst: dst, + Gw: ptr.To(networkingv1beta1.IP(remoteInterfaceIP)), }, - }) - } + }, + }) } return nil } @@ -134,19 +220,22 @@ func forgeMutateRouteConfiguration(cfg *networkingv1beta1.Configuration, // GetGatewayMode returns the mode of the Gateway related to the Configuration. func GetGatewayMode(ctx context.Context, cl client.Client, remoteClusterID liqov1beta1.ClusterID) (gateway.Mode, error) { - gwserver, gwclient, err := getters.GetGatewaysByClusterID(ctx, cl, remoteClusterID) + gwServerList, gwClientList, err := getters.GetGatewaysByClusterID(ctx, cl, remoteClusterID) if err != nil { return "", err } + hasServers := gwServerList != nil && len(gwServerList.Items) > 0 + hasClients := gwClientList != nil && len(gwClientList.Items) > 0 + switch { - case gwclient == nil && gwserver == nil: + case !hasServers && !hasClients: return "", nil - case gwclient != nil && gwserver != nil: - return "", fmt.Errorf("multiple Gateways found for cluster %s", remoteClusterID) - case gwclient == nil && gwserver != nil: + case hasServers && hasClients: + return "", fmt.Errorf("both GatewayServer and GatewayClient found for cluster %s", remoteClusterID) + case hasServers: return gateway.ModeServer, nil - case gwclient != nil && gwserver == nil: + case hasClients: return gateway.ModeClient, nil } diff --git a/pkg/liqo-controller-manager/networking/external-network/server-operator/server_controller.go b/pkg/liqo-controller-manager/networking/external-network/server-operator/server_controller.go index 8de72fed48..b9383e2b28 100644 --- a/pkg/liqo-controller-manager/networking/external-network/server-operator/server_controller.go +++ b/pkg/liqo-controller-manager/networking/external-network/server-operator/server_controller.go @@ -227,6 +227,17 @@ func (r *ServerReconciler) EnsureGatewayServer(ctx context.Context, gwServer *ne resource.AddGlobalAnnotations(objChild) + // Report the source template identity on the rendered resource so that peer gateway controllers can filter + // and compare template revisions. + annotations := objChild.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[consts.TemplateNameAnnotationKey] = template.GetName() + annotations[consts.TemplateNamespaceAnnotationKey] = template.GetNamespace() + annotations[consts.TemplateGenerationAnnotationKey] = fmt.Sprintf("%d", template.GetGeneration()) + objChild.SetAnnotations(annotations) + objChild.SetOwnerReferences([]metav1.OwnerReference{ { APIVersion: gwServer.APIVersion, diff --git a/pkg/liqo-controller-manager/networking/external-network/wireguard/rollout.go b/pkg/liqo-controller-manager/networking/external-network/wireguard/rollout.go new file mode 100644 index 0000000000..1335fbe54b --- /dev/null +++ b/pkg/liqo-controller-manager/networking/external-network/wireguard/rollout.go @@ -0,0 +1,121 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wireguard + +import ( + "context" + "fmt" + "time" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/liqotech/liqo/pkg/consts" + "github.com/liqotech/liqo/pkg/gateway" +) + +const ( + // DefaultRolloutRequeueInterval is the delay before a controller retries when + // its rollout is gated by a peer. + DefaultRolloutRequeueInterval = 5 * time.Second +) + +// IsDeploymentRollingOut reports whether the Deployment is currently rolling out. +func IsDeploymentRollingOut(dep *appsv1.Deployment) bool { + if dep == nil { + return false + } + desired := dep.Status.Replicas + if dep.Spec.Replicas != nil { + desired = *dep.Spec.Replicas + } + if dep.Generation > dep.Status.ObservedGeneration { + return true + } + return dep.Status.UpdatedReplicas < desired || + dep.Status.AvailableReplicas < desired || + dep.Status.UnavailableReplicas > 0 +} + +// ShouldDelayRollout checks whether the requested rollout of the gateway Deployment should be delayed because a peer Deployment +// for the same remote cluster is still stale or rolling out. +func ShouldDelayRollout(ctx context.Context, cl client.Client, remoteClusterID string, + self types.NamespacedName, selfTemplateName, selfTemplateNamespace, desiredGeneration string) (delay bool, reason string, err error) { + // If the source template is not known (e.g., manually created resources), we + // cannot compare peers, so we proceed without gating. + if remoteClusterID == "" || selfTemplateName == "" || selfTemplateNamespace == "" || desiredGeneration == "" { + return false, "", nil + } + + // We need to select all the peer deployments pointing to the same cluster-id and using the same template. + selector := labels.SelectorFromSet(labels.Set{ + consts.RemoteClusterID: remoteClusterID, + consts.NetworkingComponentKey: gateway.GatewayComponentGateway, + }) + + var deps appsv1.DeploymentList + if err := cl.List(ctx, &deps, &client.ListOptions{LabelSelector: selector}); err != nil { + return false, "", fmt.Errorf("listing peer gateway deployments: %w", err) + } + + selfKey := self.String() + for i := range deps.Items { + dep := &deps.Items[i] + if dep.Namespace == self.Namespace && dep.Name == self.Name { + continue + } + + // Ignore peers generated from a different source template. + annotations := dep.GetAnnotations() + if annotations[consts.TemplateNameAnnotationKey] != selfTemplateName || + annotations[consts.TemplateNamespaceAnnotationKey] != selfTemplateNamespace { + continue + } + + peerKey := client.ObjectKeyFromObject(dep).String() + peerGeneration := annotations[consts.TemplateGenerationAnnotationKey] + + // If a lower-named peer with a different generation is still waiting to roll. We must wait for it before we can proceed. + if peerKey < selfKey && peerGeneration != desiredGeneration { + return true, fmt.Sprintf("lower-named peer %s has stale generation %q (want %q)", peerKey, peerGeneration, desiredGeneration), nil + } + + // Wait while the peer that comes before us is still rolling out. + if IsDeploymentRollingOut(dep) { + return true, fmt.Sprintf("peer %s is still rolling out", peerKey), nil + } + } + + return false, "", nil +} + +func isDeploymentUpdateNeeded(existing, desired *appsv1.Deployment) bool { + if existing == nil || desired == nil { + return existing != desired + } + return !equality.Semantic.DeepEqual(existing.Spec, desired.Spec) || + !equality.Semantic.DeepEqual(existing.Labels, desired.Labels) || + !equality.Semantic.DeepEqual(existing.Annotations, desired.Annotations) +} + +func getRemoteClusterID(obj client.Object) string { + if obj == nil || obj.GetLabels() == nil { + return "" + } + return obj.GetLabels()[consts.RemoteClusterID] +} diff --git a/pkg/utils/ipc/common.go b/pkg/liqo-controller-manager/networking/external-network/wireguard/rollout_suite_test.go similarity index 74% rename from pkg/utils/ipc/common.go rename to pkg/liqo-controller-manager/networking/external-network/wireguard/rollout_suite_test.go index 0be6f7fcd7..1e5a27a119 100644 --- a/pkg/utils/ipc/common.go +++ b/pkg/liqo-controller-manager/networking/external-network/wireguard/rollout_suite_test.go @@ -12,11 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package ipc +package wireguard -import "fmt" +import ( + "testing" -// ForgeStartMessage creates a start message. -func ForgeStartMessage(id string) string { - return fmt.Sprintf("Start %s", id) + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestWireguard(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Wireguard external-network suite") } diff --git a/pkg/liqo-controller-manager/networking/external-network/wireguard/rollout_test.go b/pkg/liqo-controller-manager/networking/external-network/wireguard/rollout_test.go new file mode 100644 index 0000000000..fdc1dec231 --- /dev/null +++ b/pkg/liqo-controller-manager/networking/external-network/wireguard/rollout_test.go @@ -0,0 +1,453 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wireguard + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + networkingv1beta1 "github.com/liqotech/liqo/apis/networking/v1beta1" + "github.com/liqotech/liqo/pkg/consts" + "github.com/liqotech/liqo/pkg/gateway" + "github.com/liqotech/liqo/pkg/gateway/forge" +) + +func newTestClient(objs ...client.Object) client.Client { + scheme := runtime.NewScheme() + Expect(appsv1.AddToScheme(scheme)).To(Succeed()) + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + return fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() +} + +func newIntegrationClient(objs ...client.Object) client.Client { + scheme := runtime.NewScheme() + Expect(networkingv1beta1.AddToScheme(scheme)).To(Succeed()) + Expect(appsv1.AddToScheme(scheme)).To(Succeed()) + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + Expect(rbacv1.AddToScheme(scheme)).To(Succeed()) + Expect(monitoringv1.AddToScheme(scheme)).To(Succeed()) + return fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objs...). + WithStatusSubresource(&networkingv1beta1.WgGatewayServer{}, &appsv1.Deployment{}). + Build() +} + +func makeDeployment(name, remoteClusterID, templateName, templateNamespace, generation string, rolling bool) *appsv1.Deployment { + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "ns", + Generation: 1, + Labels: map[string]string{ + consts.RemoteClusterID: remoteClusterID, + consts.NetworkingComponentKey: gateway.GatewayComponentGateway, + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(1)), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": name}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "gateway", Image: "liqo/gateway:latest"}, + }, + }, + }, + }, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 1, + Replicas: 1, + UpdatedReplicas: 1, + AvailableReplicas: 1, + UnavailableReplicas: 0, + }, + } + if templateName != "" || templateNamespace != "" || generation != "" { + if dep.Annotations == nil { + dep.Annotations = map[string]string{} + } + if templateName != "" { + dep.Annotations[consts.TemplateNameAnnotationKey] = templateName + } + if templateNamespace != "" { + dep.Annotations[consts.TemplateNamespaceAnnotationKey] = templateNamespace + } + if generation != "" { + dep.Annotations[consts.TemplateGenerationAnnotationKey] = generation + } + } + if rolling { + dep.Status.ObservedGeneration = 0 + } + return dep +} + +func buildWgGatewayServer(name, namespace, remoteClusterID string) *networkingv1beta1.WgGatewayServer { + return &networkingv1beta1.WgGatewayServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + UID: types.UID(name + "-uid"), + Labels: map[string]string{ + consts.RemoteClusterID: remoteClusterID, + }, + Annotations: map[string]string{ + consts.TemplateNameAnnotationKey: "server-template", + consts.TemplateNamespaceAnnotationKey: "server-template-ns", + consts.TemplateGenerationAnnotationKey: "2", + }, + Finalizers: []string{consts.ClusterRoleBindingFinalizer}, + }, + Spec: networkingv1beta1.WgGatewayServerSpec{ + SecretRef: corev1.LocalObjectReference{ + Name: name + "-keys", + }, + Service: networkingv1beta1.ServiceTemplate{ + Metadata: metav1.ObjectMeta{ + Labels: map[string]string{ + consts.RemoteClusterID: remoteClusterID, + consts.NetworkingComponentKey: gateway.GatewayComponentGateway, + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "10.0.0.1", + ClusterIPs: []string{"10.0.0.1"}, + Ports: []corev1.ServicePort{ + {Port: 51820, Protocol: corev1.ProtocolUDP}, + }, + }, + }, + Deployment: networkingv1beta1.DeploymentTemplate{ + Metadata: metav1.ObjectMeta{ + Labels: map[string]string{ + consts.RemoteClusterID: remoteClusterID, + consts.NetworkingComponentKey: gateway.GatewayComponentGateway, + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": forge.GatewayResourceName(name), + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": forge.GatewayResourceName(name), + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "wireguard", + Image: "liqo/gateway-wireguard:latest", + Env: []corev1.EnvVar{ + {Name: "VERSION", Value: "new"}, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func buildExistingDeployment(wgServer *networkingv1beta1.WgGatewayServer) *appsv1.Deployment { + desired := wgServer.Spec.Deployment.Spec.DeepCopy() + desired.Template.Spec.Containers[0].Env[0].Value = "old" + depName := forge.GatewayResourceName(wgServer.Name) + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: depName, + Namespace: wgServer.Namespace, + Generation: 1, + Labels: map[string]string{ + consts.RemoteClusterID: wgServer.Labels[consts.RemoteClusterID], + consts.NetworkingComponentKey: gateway.GatewayComponentGateway, + }, + Annotations: map[string]string{ + consts.TemplateNameAnnotationKey: wgServer.Annotations[consts.TemplateNameAnnotationKey], + consts.TemplateNamespaceAnnotationKey: wgServer.Annotations[consts.TemplateNamespaceAnnotationKey], + consts.TemplateGenerationAnnotationKey: "1", + }, + }, + Spec: *desired, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 1, + Replicas: 1, + UpdatedReplicas: 1, + AvailableReplicas: 1, + UnavailableReplicas: 0, + }, + } +} + +func buildService(wgServer *networkingv1beta1.WgGatewayServer) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: forge.GatewayResourceName(wgServer.Name), + Namespace: wgServer.Namespace, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "10.0.0.1", + ClusterIPs: []string{"10.0.0.1"}, + Ports: []corev1.ServicePort{ + {Port: 51820, Protocol: corev1.ProtocolUDP}, + }, + }, + } +} + +func buildActivePod(wgServer *networkingv1beta1.WgGatewayServer) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: forge.GatewayResourceName(wgServer.Name) + "-abc", + Namespace: wgServer.Namespace, + Labels: map[string]string{ + "app": forge.GatewayResourceName(wgServer.Name), + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "wireguard", Image: "liqo/gateway-wireguard:latest"}, + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + PodIP: "10.244.1.10", + }, + } +} + +func buildSecret(wgServer *networkingv1beta1.WgGatewayServer) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: wgServer.Spec.SecretRef.Name, + Namespace: wgServer.Namespace, + Labels: map[string]string{ + consts.RemoteClusterID: wgServer.Labels[consts.RemoteClusterID], + consts.GatewayResourceLabel: consts.GatewayResourceLabelValue, + consts.NetworkingComponentKey: gateway.GatewayComponentGateway, + }, + }, + Data: map[string][]byte{ + consts.PrivateKeyField: []byte("private-key"), + consts.PublicKeyField: []byte("public-key"), + }, + } +} + +var _ = Describe("Rollout helpers", func() { + Context("IsDeploymentRollingOut", func() { + DescribeTable("reports whether a Deployment is rolling out", + func(dep *appsv1.Deployment, expected bool) { + Expect(IsDeploymentRollingOut(dep)).To(Equal(expected)) + }, + Entry("nil deployment", nil, false), + Entry("steady state", makeDeployment("gw", "remote", "template", "template-ns", "generation", false), false), + Entry("unobserved generation", func() *appsv1.Deployment { + d := makeDeployment("gw", "remote", "template", "template-ns", "generation", false) + d.Generation = 2 + d.Status.ObservedGeneration = 1 + return d + }(), true), + Entry("updated replicas below desired", func() *appsv1.Deployment { + d := makeDeployment("gw", "remote", "template", "template-ns", "generation", false) + d.Status.UpdatedReplicas = 0 + return d + }(), true), + Entry("available replicas below desired", func() *appsv1.Deployment { + d := makeDeployment("gw", "remote", "template", "template-ns", "generation", false) + d.Status.AvailableReplicas = 0 + return d + }(), true), + Entry("unavailable replicas present", func() *appsv1.Deployment { + d := makeDeployment("gw", "remote", "template", "template-ns", "generation", false) + d.Status.UnavailableReplicas = 1 + return d + }(), true), + ) + }) + + Context("ShouldDelayRollout", func() { + const ( + remoteClusterID = "remote" + desiredTemplateName = "template" + desiredTemplateNamespace = "template-ns" + desiredGeneration = "desired-generation" + ) + + DescribeTable("decides whether a rollout should be delayed", + func(self types.NamespacedName, peers []*appsv1.Deployment, expected bool) { + objs := make([]client.Object, len(peers)) + for i := range peers { + objs[i] = peers[i] + } + cl := newTestClient(objs...) + delay, _, err := ShouldDelayRollout(context.Background(), cl, remoteClusterID, self, + desiredTemplateName, desiredTemplateNamespace, desiredGeneration) + Expect(err).NotTo(HaveOccurred()) + Expect(delay).To(Equal(expected)) + }, + Entry("no peers", types.NamespacedName{Namespace: "ns", Name: "gw-a"}, nil, false), + Entry("self is excluded from peer list", + types.NamespacedName{Namespace: "ns", Name: "gw-a"}, + []*appsv1.Deployment{makeDeployment("gw-a", remoteClusterID, desiredTemplateName, desiredTemplateNamespace, desiredGeneration, false)}, + false, + ), + Entry("lower-named peer with stale generation delays", + types.NamespacedName{Namespace: "ns", Name: "gw-b"}, + []*appsv1.Deployment{makeDeployment("gw-a", remoteClusterID, desiredTemplateName, desiredTemplateNamespace, "old-generation", false)}, + true, + ), + Entry("lower-named peer with desired generation but rolling out delays", + types.NamespacedName{Namespace: "ns", Name: "gw-b"}, + []*appsv1.Deployment{makeDeployment("gw-a", remoteClusterID, desiredTemplateName, desiredTemplateNamespace, desiredGeneration, true)}, + true, + ), + Entry("lower-named peer up to date does not delay", + types.NamespacedName{Namespace: "ns", Name: "gw-b"}, + []*appsv1.Deployment{makeDeployment("gw-a", remoteClusterID, desiredTemplateName, desiredTemplateNamespace, desiredGeneration, false)}, + false, + ), + Entry("higher-named peer with stale generation does not delay", + types.NamespacedName{Namespace: "ns", Name: "gw-a"}, + []*appsv1.Deployment{makeDeployment("gw-b", remoteClusterID, desiredTemplateName, desiredTemplateNamespace, "old-generation", false)}, + false, + ), + Entry("higher-named peer rolling out delays", + types.NamespacedName{Namespace: "ns", Name: "gw-a"}, + []*appsv1.Deployment{makeDeployment("gw-b", remoteClusterID, desiredTemplateName, desiredTemplateNamespace, desiredGeneration, true)}, + true, + ), + Entry("peer with different remote cluster id is ignored", + types.NamespacedName{Namespace: "ns", Name: "gw-a"}, + []*appsv1.Deployment{makeDeployment("gw-b", "other-cluster", desiredTemplateName, desiredTemplateNamespace, "old-generation", true)}, + false, + ), + Entry("peer with same remote cluster id but different template name is ignored", + types.NamespacedName{Namespace: "ns", Name: "gw-b"}, + []*appsv1.Deployment{makeDeployment("gw-a", remoteClusterID, "other-template", desiredTemplateNamespace, "old-generation", true)}, + false, + ), + Entry("peer with same remote cluster id and name but different template namespace is ignored", + types.NamespacedName{Namespace: "ns", Name: "gw-b"}, + []*appsv1.Deployment{makeDeployment("gw-a", remoteClusterID, desiredTemplateName, "other-namespace", "old-generation", true)}, + false, + ), + ) + }) +}) + +var _ = Describe("WgGatewayServer controller serialization gate", Ordered, func() { + const remoteClusterID = "remote" + + var ( + ctx context.Context + cl client.Client + r *WgGatewayServerReconciler + wgA *networkingv1beta1.WgGatewayServer + wgB *networkingv1beta1.WgGatewayServer + depA *appsv1.Deployment + depB *appsv1.Deployment + updatedA appsv1.Deployment + ) + + BeforeAll(func() { + ctx = context.Background() + + wgA = buildWgGatewayServer("a", "test-a", remoteClusterID) + wgB = buildWgGatewayServer("b", "test-b", remoteClusterID) + depA = buildExistingDeployment(wgA) + depB = buildExistingDeployment(wgB) + + cl = newIntegrationClient( + wgA, wgB, + depA, depB, + buildService(wgA), buildService(wgB), + buildActivePod(wgA), buildActivePod(wgB), + buildSecret(wgA), buildSecret(wgB), + ) + + recorder := record.NewFakeRecorder(100) + r = NewWgGatewayServerReconciler(cl, cl.Scheme(), recorder, "liqo-gateway") + }) + + It("allows the lowest-named peer to proceed and update its Deployment", func() { + res, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: wgA.Namespace, Name: wgA.Name}}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.RequeueAfter).To(BeZero()) + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(depA), &updatedA)).To(Succeed()) + Expect(updatedA.Spec.Template.Spec.Containers[0].Env[0].Value).To(Equal("new")) + Expect(updatedA.Annotations).To(HaveKey(consts.TemplateNameAnnotationKey)) + Expect(updatedA.Annotations[consts.TemplateNameAnnotationKey]).To(Equal("server-template")) + Expect(updatedA.Annotations).To(HaveKey(consts.TemplateNamespaceAnnotationKey)) + Expect(updatedA.Annotations[consts.TemplateNamespaceAnnotationKey]).To(Equal("server-template-ns")) + Expect(updatedA.Annotations).To(HaveKey(consts.TemplateGenerationAnnotationKey)) + Expect(updatedA.Annotations[consts.TemplateGenerationAnnotationKey]).To(Equal("2")) + }) + + It("delays the higher-named peer while the lower-named peer is rolling out", func() { + updatedA.Generation = 2 + Expect(cl.Update(ctx, &updatedA)).To(Succeed()) + + res, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: wgB.Namespace, Name: wgB.Name}}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.RequeueAfter).To(BeNumerically(">", 0)) + + var unchangedB appsv1.Deployment + Expect(cl.Get(ctx, client.ObjectKeyFromObject(depB), &unchangedB)).To(Succeed()) + Expect(unchangedB.Spec.Template.Spec.Containers[0].Env[0].Value).To(Equal("old")) + }) + + It("allows the higher-named peer to roll after the lower-named peer finishes", func() { + updatedA.Status.ObservedGeneration = updatedA.Generation + updatedA.Status.UpdatedReplicas = 1 + updatedA.Status.AvailableReplicas = 1 + updatedA.Status.UnavailableReplicas = 0 + Expect(cl.Status().Update(ctx, &updatedA)).To(Succeed()) + + res, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: wgB.Namespace, Name: wgB.Name}}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.RequeueAfter).To(BeZero()) + + var updatedB appsv1.Deployment + Expect(cl.Get(ctx, client.ObjectKeyFromObject(depB), &updatedB)).To(Succeed()) + Expect(updatedB.Spec.Template.Spec.Containers[0].Env[0].Value).To(Equal("new")) + }) +}) diff --git a/pkg/liqo-controller-manager/networking/external-network/wireguard/utils.go b/pkg/liqo-controller-manager/networking/external-network/wireguard/utils.go index d8235ae668..6ab95b4341 100644 --- a/pkg/liqo-controller-manager/networking/external-network/wireguard/utils.go +++ b/pkg/liqo-controller-manager/networking/external-network/wireguard/utils.go @@ -246,8 +246,8 @@ func getWireGuardSecret(ctx context.Context, cl client.Client, wgObj metav1.Obje } } -func listActiveGatewayPod(ctx context.Context, cl client.Client, namespace string) (*corev1.Pod, error) { - podsSelector := client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(gateway.ForgeActiveGatewayPodLabels())} +func listActiveGatewayPod(ctx context.Context, cl client.Client, namespace string, selector labels.Selector) (*corev1.Pod, error) { + podsSelector := client.MatchingLabelsSelector{Selector: selector} var podList corev1.PodList if err := cl.List(ctx, &podList, client.InNamespace(namespace), podsSelector); err != nil { klog.Errorf("Unable to list active gateway pods in namespace %q: %v", namespace, err) diff --git a/pkg/liqo-controller-manager/networking/external-network/wireguard/wggatewayclient_controller.go b/pkg/liqo-controller-manager/networking/external-network/wireguard/wggatewayclient_controller.go index 838e67f2cb..73c0fda44d 100644 --- a/pkg/liqo-controller-manager/networking/external-network/wireguard/wggatewayclient_controller.go +++ b/pkg/liqo-controller-manager/networking/external-network/wireguard/wggatewayclient_controller.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" @@ -185,6 +186,33 @@ func (r *WgGatewayClientReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, err } + // If the deployment already exists and its desired spec differs from the current one update the deployment making sure + // to respect the rollout gate. + if deploy != nil { + desired := deploy.DeepCopy() + if err := r.mutateFnWgClientDeployment(desired, wgClient); err != nil { + return ctrl.Result{}, fmt.Errorf("computing desired deployment %q: %w", deployNsName, err) + } + + if isDeploymentUpdateNeeded(deploy, desired) { + remoteClusterID := getRemoteClusterID(deploy) + desiredTemplateName := wgClient.GetAnnotations()[consts.TemplateNameAnnotationKey] + desiredTemplateNamespace := wgClient.GetAnnotations()[consts.TemplateNamespaceAnnotationKey] + desiredGeneration := wgClient.GetAnnotations()[consts.TemplateGenerationAnnotationKey] + delay, reason, err := ShouldDelayRollout(ctx, r.Client, remoteClusterID, + types.NamespacedName{Namespace: deploy.Namespace, Name: deploy.Name}, + desiredTemplateName, desiredTemplateNamespace, desiredGeneration) + if err != nil { + return ctrl.Result{}, fmt.Errorf("checking rollout gate for %q: %w", deployNsName, err) + } + if delay { + r.eventRecorder.Eventf(wgClient, corev1.EventTypeNormal, "DeploymentUpdateDelayed", + "Deployment update delayed: %s", reason) + return ctrl.Result{RequeueAfter: DefaultRolloutRequeueInterval}, nil + } + } + } + // Ensure deployment (create or update) _, err = r.ensureDeployment(ctx, wgClient, deployNsName) if err != nil { @@ -246,6 +274,24 @@ func (r *WgGatewayClientReconciler) mutateFnWgClientDeployment(deployment *appsv // Forge spec deployment.Spec = wgClient.Spec.Deployment.Spec + // ShouldDelayRollout uses them to consider only peers that originate from the + // same template (including namespace). + if deployment.Annotations == nil { + deployment.Annotations = map[string]string{} + } + if templateName := wgClient.GetAnnotations()[consts.TemplateNameAnnotationKey]; templateName != "" { + deployment.Annotations[consts.TemplateNameAnnotationKey] = templateName + } + if templateNamespace := wgClient.GetAnnotations()[consts.TemplateNamespaceAnnotationKey]; templateNamespace != "" { + deployment.Annotations[consts.TemplateNamespaceAnnotationKey] = templateNamespace + } + + // Stamp the desired template generation so that peer controllers can detect + // whether this Deployment has already rolled to the current template revision. + if templateGeneration := wgClient.GetAnnotations()[consts.TemplateGenerationAnnotationKey]; templateGeneration != "" { + deployment.Annotations[consts.TemplateGenerationAnnotationKey] = templateGeneration + } + if wgClient.Status.SecretRef != nil { // When no secret reference is provided, we will need to replace the secret name in the deployment manifest with the auto-generated one. for i := range deployment.Spec.Template.Spec.Volumes { @@ -288,7 +334,7 @@ func (r *WgGatewayClientReconciler) handleInternalEndpointStatus(ctx context.Con return nil } - gwPod, err := listActiveGatewayPod(ctx, r.Client, dep.Namespace) + gwPod, err := listActiveGatewayPod(ctx, r.Client, dep.Namespace, labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)) if err != nil { return fmt.Errorf("retrieving active gateway pods: %w", err) } diff --git a/pkg/liqo-controller-manager/networking/external-network/wireguard/wggatewayserver_controller.go b/pkg/liqo-controller-manager/networking/external-network/wireguard/wggatewayserver_controller.go index 67432da009..47f9e346b0 100644 --- a/pkg/liqo-controller-manager/networking/external-network/wireguard/wggatewayserver_controller.go +++ b/pkg/liqo-controller-manager/networking/external-network/wireguard/wggatewayserver_controller.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" @@ -195,6 +196,32 @@ func (r *WgGatewayServerReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, err } + // If the deployment already exists and its desired spec differs from the current one update the deployment making sure + // to respect the rollout gate. + if deploy != nil { + desired := deploy.DeepCopy() + if err := r.mutateFnWgServerDeployment(desired, wgServer); err != nil { + return ctrl.Result{}, fmt.Errorf("computing desired deployment %q: %w", deployNsName, err) + } + if isDeploymentUpdateNeeded(deploy, desired) { + remoteClusterID := getRemoteClusterID(deploy) + desiredTemplateName := wgServer.GetAnnotations()[consts.TemplateNameAnnotationKey] + desiredTemplateNamespace := wgServer.GetAnnotations()[consts.TemplateNamespaceAnnotationKey] + desiredGeneration := wgServer.GetAnnotations()[consts.TemplateGenerationAnnotationKey] + delay, reason, err := ShouldDelayRollout(ctx, r.Client, remoteClusterID, + types.NamespacedName{Namespace: deploy.Namespace, Name: deploy.Name}, + desiredTemplateName, desiredTemplateNamespace, desiredGeneration) + if err != nil { + return ctrl.Result{}, fmt.Errorf("checking rollout gate for %q: %w", deployNsName, err) + } + if delay { + r.eventRecorder.Eventf(wgServer, corev1.EventTypeNormal, "DeploymentUpdateDelayed", + "Deployment update delayed: %s", reason) + return ctrl.Result{RequeueAfter: DefaultRolloutRequeueInterval}, nil + } + } + } + // Ensure deployment (create or update) _, err = r.ensureDeployment(ctx, wgServer, deployNsName) if err != nil { @@ -283,6 +310,22 @@ func (r *WgGatewayServerReconciler) mutateFnWgServerDeployment(deployment *appsv // Forge spec deployment.Spec = wgServer.Spec.Deployment.Spec + // Copy the source template identity annotations from the WgGatewayServer object. + // ShouldDelayRollout uses them to consider only peers that originate from the + // same template (including namespace). + if deployment.Annotations == nil { + deployment.Annotations = map[string]string{} + } + if templateName := wgServer.GetAnnotations()[consts.TemplateNameAnnotationKey]; templateName != "" { + deployment.Annotations[consts.TemplateNameAnnotationKey] = templateName + } + if templateNamespace := wgServer.GetAnnotations()[consts.TemplateNamespaceAnnotationKey]; templateNamespace != "" { + deployment.Annotations[consts.TemplateNamespaceAnnotationKey] = templateNamespace + } + if templateGeneration := wgServer.GetAnnotations()[consts.TemplateGenerationAnnotationKey]; templateGeneration != "" { + deployment.Annotations[consts.TemplateGenerationAnnotationKey] = templateGeneration + } + if wgServer.Status.SecretRef != nil { for i := range deployment.Spec.Template.Spec.Volumes { if deployment.Spec.Template.Spec.Volumes[i].Name == wireguardVolumeName { @@ -384,7 +427,7 @@ func (r *WgGatewayServerReconciler) forgeEndpointStatusNodePort(ctx context.Cont port := service.Spec.Ports[0].NodePort protocol := &service.Spec.Ports[0].Protocol - pod, err := listActiveGatewayPod(ctx, r.Client, dep.Namespace) + pod, err := listActiveGatewayPod(ctx, r.Client, dep.Namespace, labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)) if err != nil { return nil, fmt.Errorf("retrieving active gateway pod: %w", err) } @@ -454,7 +497,7 @@ func (r *WgGatewayServerReconciler) handleInternalEndpointStatus(ctx context.Con return nil } - gwPod, err := listActiveGatewayPod(ctx, r.Client, dep.Namespace) + gwPod, err := listActiveGatewayPod(ctx, r.Client, dep.Namespace, labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)) if err != nil { return fmt.Errorf("retrieving active gateway pods: %w", err) } diff --git a/pkg/liqo-controller-manager/networking/internal-network/client-controller/client_controller.go b/pkg/liqo-controller-manager/networking/internal-network/client-controller/client_controller.go index 3424a78c38..8878dc2032 100644 --- a/pkg/liqo-controller-manager/networking/internal-network/client-controller/client_controller.go +++ b/pkg/liqo-controller-manager/networking/internal-network/client-controller/client_controller.go @@ -110,14 +110,9 @@ func (r *ClientReconciler) ensureInternalFabric(ctx context.Context, gwClient *n return fmt.Errorf("internal endpoint not found for the gateway client %q", gwClient.Name) } - internalFabricName, err := netutils.ForgeInternalFabricName(ctx, r.Client, &gwClient.ObjectMeta) - if err != nil { - return fmt.Errorf("unable to retrieve the cluster ID from the gateway client %q", gwClient.Name) - } - internalFabric := &networkingv1beta1.InternalFabric{ ObjectMeta: metav1.ObjectMeta{ - Name: internalFabricName, + Name: gwClient.Name, Namespace: gwClient.Namespace, }, } @@ -179,15 +174,18 @@ func (r *ClientReconciler) gatewayClientEnqueuerByRemoteID() handler.MapFunc { return nil } - gwClient, err := getters.GetGatewayClientByClusterID(ctx, r.Client, remoteClusterID, corev1.NamespaceAll) + gwClientList, err := getters.ListGatewayClientsByClusterID(ctx, r.Client, remoteClusterID, corev1.NamespaceAll) if err != nil { - if apierrors.IsNotFound(err) { - return nil - } - klog.Errorf("unable to get the gateway client for cluster %s: %s", remoteClusterID, err) + klog.Errorf("unable to list gateway clients for cluster %s: %s", remoteClusterID, err) return nil } - return []reconcile.Request{{NamespacedName: client.ObjectKeyFromObject(gwClient)}} + var requests []reconcile.Request + for i := range gwClientList.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(&gwClientList.Items[i]), + }) + } + return requests } } diff --git a/pkg/liqo-controller-manager/networking/internal-network/id/ecmpmark.go b/pkg/liqo-controller-manager/networking/internal-network/id/ecmpmark.go new file mode 100644 index 0000000000..046449e365 --- /dev/null +++ b/pkg/liqo-controller-manager/networking/internal-network/id/ecmpmark.go @@ -0,0 +1,67 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package id + +import ( + "context" + "fmt" + "math" + "sync" + + "k8s.io/apimachinery/pkg/util/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + networkingv1beta1 "github.com/liqotech/liqo/apis/networking/v1beta1" + "github.com/liqotech/liqo/pkg/consts" +) + +var ecmpMarkManager *Manager[uint32] +var ecmpMarkOnce sync.Once + +// ResetECMPMarkManager resets the singleton ECMP mark manager. It is intended for tests only. +func ResetECMPMarkManager() { + ecmpMarkManager = nil + ecmpMarkOnce = sync.Once{} +} + +// GetECMPMarkManager returns the Manager for ECMP policy-routing marks or creates it if it does not exist. +// It is a singleton and recovers previously allocated marks from InternalFabric status on first use. +func GetECMPMarkManager(ctx context.Context, cl client.Client) *Manager[uint32] { + ecmpMarkOnce.Do(func() { + // Allocate marks in the dedicated Liqo ECMP range [base, base + 2^24). + ecmpMarkManager = NewWithRange[uint32](consts.ECMPReplicaMarkBase, consts.ECMPReplicaMarkBase+(1<<24)) + + var fabricList networkingv1beta1.InternalFabricList + err := cl.List(ctx, &fabricList) + runtime.Must(err) + + for i := range fabricList.Items { + fabric := &fabricList.Items[i] + if fabric.Status.ECMPMark != nil { + mark := *fabric.Status.ECMPMark + if mark < 0 || mark > math.MaxUint32 { + panic(fmt.Sprintf("invalid ECMP mark %d for InternalFabric %q: out of uint32 range", + mark, client.ObjectKeyFromObject(fabric).String())) + } + + markUint32 := uint32(mark) + err = ecmpMarkManager.Configure(client.ObjectKeyFromObject(fabric).String(), markUint32) + runtime.Must(err) + } + } + }) + + return ecmpMarkManager +} diff --git a/pkg/liqo-controller-manager/networking/internal-network/id/ecmpmark_test.go b/pkg/liqo-controller-manager/networking/internal-network/id/ecmpmark_test.go new file mode 100644 index 0000000000..d79d92bf43 --- /dev/null +++ b/pkg/liqo-controller-manager/networking/internal-network/id/ecmpmark_test.go @@ -0,0 +1,103 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package id + +import ( + "context" + "sync" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + networkingv1beta1 "github.com/liqotech/liqo/apis/networking/v1beta1" + "github.com/liqotech/liqo/pkg/consts" +) + +var _ = Describe("ECMP Mark Manager Tests", func() { + + var ( + cl client.Client + ctx context.Context + scheme *runtime.Scheme + markMgr *Manager[uint32] + ) + + BeforeEach(func() { + ctx = context.Background() + scheme = runtime.NewScheme() + Expect(networkingv1beta1.SchemeBuilder.AddToScheme(scheme)).To(Succeed()) + + // Build a fake client with an InternalFabric that already has a mark. + cl = fake.NewClientBuilder().WithScheme(scheme).WithObjects( + &networkingv1beta1.InternalFabric{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fabric-with-mark", + Namespace: corev1.NamespaceDefault, + }, + Status: networkingv1beta1.InternalFabricStatus{ + ECMPMark: ptr.To(consts.ECMPReplicaMarkBase + 42), + }, + }, + &networkingv1beta1.InternalFabric{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fabric-without-mark", + Namespace: corev1.NamespaceDefault, + }, + }, + ).Build() + + // Reset the singleton so each test starts fresh. + ecmpMarkManager = nil + ecmpMarkOnce = sync.Once{} + markMgr = GetECMPMarkManager(ctx, cl) + }) + + It("should recover existing marks from InternalFabric status", func() { + mark, err := markMgr.Allocate("default/fabric-with-mark") + Expect(err).To(Succeed()) + Expect(mark).To(BeNumerically("==", consts.ECMPReplicaMarkBase+42)) + }) + + It("should allocate marks in the ECMP range", func() { + mark, err := markMgr.Allocate("default/fabric-without-mark") + Expect(err).To(Succeed()) + Expect(mark).To(BeNumerically(">=", consts.ECMPReplicaMarkBase)) + Expect(mark).To(BeNumerically("<", consts.ECMPReplicaMarkBase+(1<<24))) + }) + + It("should allocate different marks for different fabrics", func() { + mark1, err := markMgr.Allocate("default/fabric-without-mark") + Expect(err).To(Succeed()) + + cl2 := fake.NewClientBuilder().WithScheme(scheme).WithObjects( + &networkingv1beta1.InternalFabric{ + ObjectMeta: metav1.ObjectMeta{ + Name: "another-fabric", + Namespace: corev1.NamespaceDefault, + }, + }, + ).Build() + + mark2, err := GetECMPMarkManager(ctx, cl2).Allocate("default/another-fabric") + Expect(err).To(Succeed()) + Expect(mark1).To(Not(Equal(mark2))) + }) +}) diff --git a/pkg/liqo-controller-manager/networking/internal-network/id/id_test.go b/pkg/liqo-controller-manager/networking/internal-network/id/id_test.go index 7855c38bdb..eef18f5e05 100644 --- a/pkg/liqo-controller-manager/networking/internal-network/id/id_test.go +++ b/pkg/liqo-controller-manager/networking/internal-network/id/id_test.go @@ -158,4 +158,37 @@ var _ = Describe("ID Tests", func() { }) + Context("with a ranged Manager", func() { + + It("should allocate IDs inside the range", func() { + manager := NewWithRange[uint32](100, 104) + id, err := manager.Allocate("test") + Expect(err).To(Succeed()) + Expect(id).To(BeNumerically(">=", 100)) + Expect(id).To(BeNumerically("<", 104)) + }) + + It("should not allocate IDs outside the range", func() { + manager := NewWithRange[uint32](100, 103) + _, err := manager.Allocate("a") + Expect(err).To(Succeed()) + _, err = manager.Allocate("b") + Expect(err).To(Succeed()) + _, err = manager.Allocate("c") + Expect(err).To(Succeed()) + _, err = manager.Allocate("d") + Expect(err).To(HaveOccurred()) + }) + + It("should configure IDs inside the range", func() { + manager := NewWithRange[uint32](100, 200) + err := manager.Configure("test", 150) + Expect(err).To(Succeed()) + id, err := manager.Allocate("test") + Expect(err).To(Succeed()) + Expect(id).To(BeNumerically("==", 150)) + }) + + }) + }) diff --git a/pkg/liqo-controller-manager/networking/internal-network/id/manager.go b/pkg/liqo-controller-manager/networking/internal-network/id/manager.go index 231b9b5f32..23d0b4c44e 100644 --- a/pkg/liqo-controller-manager/networking/internal-network/id/manager.go +++ b/pkg/liqo-controller-manager/networking/internal-network/id/manager.go @@ -29,23 +29,29 @@ type Manager[T Integer] struct { allocatedMutex sync.Mutex allocated map[string]T // map[name]id allocatedReverse map[T]string // map[id]name + minID T nextAllocatedID T maxID T } -// New returns a new Manager. +// New returns a new Manager using IDs in the range [0, 2^24). func New[T Integer]() *Manager[T] { - allocated := make(map[string]T) - allocatedReverse := make(map[T]string) - // only 24 bits are used maxID := (1 << 24) - 1 + return NewWithRange[T](0, T(maxID)) +} + +// NewWithRange returns a new Manager using IDs in the range [minID, maxID). +func NewWithRange[T Integer](minID, maxID T) *Manager[T] { + allocated := make(map[string]T) + allocatedReverse := make(map[T]string) return &Manager[T]{ allocated: allocated, allocatedReverse: allocatedReverse, - nextAllocatedID: 0, - maxID: T(maxID), + minID: minID, + nextAllocatedID: minID, + maxID: maxID, } } @@ -74,10 +80,11 @@ func (m *Manager[T]) Allocate(name string) (T, error) { } var id T + // First pass: search from the next candidate up to maxID. for id = m.nextAllocatedID; id < m.maxID; id++ { if _, ok := m.allocatedReverse[id]; !ok { if id == m.maxID-1 { - m.nextAllocatedID = 0 + m.nextAllocatedID = m.minID } else { m.nextAllocatedID = id + 1 } @@ -86,10 +93,11 @@ func (m *Manager[T]) Allocate(name string) (T, error) { return id, nil } } - for id = 0; id < m.maxID; id++ { + // Second pass: wrap around and search from minID up to nextAllocatedID. + for id = m.minID; id < m.nextAllocatedID; id++ { if _, ok := m.allocatedReverse[id]; !ok { if id == m.maxID-1 { - m.nextAllocatedID = 0 + m.nextAllocatedID = m.minID } else { m.nextAllocatedID = id + 1 } diff --git a/pkg/liqo-controller-manager/networking/internal-network/internalfabric-controller/firewall.go b/pkg/liqo-controller-manager/networking/internal-network/internalfabric-controller/firewall.go new file mode 100644 index 0000000000..a77aaf9ec5 --- /dev/null +++ b/pkg/liqo-controller-manager/networking/internal-network/internalfabric-controller/firewall.go @@ -0,0 +1,179 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internalfabriccontroller + +import ( + "context" + "fmt" + "slices" + "strings" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + networkingv1beta1 "github.com/liqotech/liqo/apis/networking/v1beta1" + "github.com/liqotech/liqo/apis/networking/v1beta1/firewall" + "github.com/liqotech/liqo/pkg/consts" + "github.com/liqotech/liqo/pkg/fabric" + "github.com/liqotech/liqo/pkg/utils/resource" +) + +// GenerateFirewallConfigurationName returns the name of the FirewallConfiguration associated to the InternalFabric. +// It uses the remote-cluster-id from the InternalFabric labels so all siblings share the same name. +func generateFirewallConfigurationName(remoteClusterID string) string { + return fmt.Sprintf("%s-node-gw", remoteClusterID) +} + +func (r *InternalFabricReconciler) ensureFirewallConfiguration( + ctx context.Context, + internalFabric *networkingv1beta1.InternalFabric, + siblings []networkingv1beta1.InternalFabric, +) error { + remoteClusterID, ok := internalFabric.Labels[consts.RemoteClusterID] + if !ok || remoteClusterID == "" { + return fmt.Errorf("InternalFabric %q does not have a remote-cluster-id label", internalFabric.Name) + } + + fw := &networkingv1beta1.FirewallConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: generateFirewallConfigurationName(remoteClusterID), + Namespace: internalFabric.Namespace, + }, + } + + // Apply the connection-tracking mark firewall only when multiple gateways are configured. + if len(siblings) < 2 { + if err := r.Delete(ctx, fw); err != nil && !k8serrors.IsNotFound(err) { + return fmt.Errorf("deleting FirewallConfiguration %q: %w", fw.Name, err) + } + return nil + } + + _, err := resource.CreateOrUpdate(ctx, r.Client, fw, func() error { + if fw.Labels == nil { + fw.Labels = make(labels.Set) + } + fw.SetLabels(labels.Merge(fw.Labels, fabric.ForgeFirewallTargetLabels())) + + fw.Spec.Table.Name = ptr.To(fw.Name) + fw.Spec.Table.Family = ptr.To(firewall.TableFamilyIPv4) + chains, err := forgeFirewallChains(internalFabric, siblings) + if err != nil { + return err + } + fw.Spec.Table.Chains = chains + + return controllerutil.SetOwnerReference(internalFabric, fw, r.Scheme) + }) + if err != nil { + return fmt.Errorf("creating or updating FirewallConfiguration %q: %w", fw.Name, err) + } + + return nil +} + +func forgeFirewallChains(internalFabric *networkingv1beta1.InternalFabric, fabrics []networkingv1beta1.InternalFabric) ([]firewall.Chain, error) { + if len(fabrics) <= 1 { + return nil, nil + } + + remoteCIDRs := slices.Clone(internalFabric.Spec.RemoteCIDRs) + slices.Sort(remoteCIDRs) + + if len(remoteCIDRs) == 0 { + return nil, nil + } + + forwardRules := make([]firewall.FilterRule, 0, len(fabrics)*len(remoteCIDRs)) + preroutingRules := make([]firewall.FilterRule, 0, len(remoteCIDRs)) + + for _, remoteCIDR := range remoteCIDRs { + for i := range fabrics { + fab := &fabrics[i] + mark, err := fabricECMPMark(fab) + if err != nil { + return nil, err + } + forwardRules = append(forwardRules, firewall.FilterRule{ + Name: ptr.To(fmt.Sprintf("mark-in-gw-%d-%s", i, sanitizeCIDR(remoteCIDR))), + Action: firewall.ActionCtMark, + Value: ptr.To(fmt.Sprintf("%d", mark)), + Counter: true, + Match: []firewall.Match{ + { + Op: firewall.MatchOperationEq, + IP: &firewall.MatchIP{ + Position: firewall.MatchPositionSrc, + Value: remoteCIDR.String(), + }, + }, + { + Op: firewall.MatchOperationEq, + Dev: &firewall.MatchDev{ + Position: firewall.MatchDevPositionIn, + Value: fab.Spec.Interface.Node.Name, + }, + }, + }, + }) + } + + preroutingRules = append(preroutingRules, firewall.FilterRule{ + Name: ptr.To(fmt.Sprintf("restore-mark-%s", sanitizeCIDR(remoteCIDR))), + Action: firewall.ActionSetMetaMarkFromCtMark, + Counter: true, + Match: []firewall.Match{ + { + Op: firewall.MatchOperationEq, + IP: &firewall.MatchIP{ + Position: firewall.MatchPositionDst, + Value: remoteCIDR.String(), + }, + }, + }, + }) + } + + return []firewall.Chain{ + { + Name: ptr.To("ecmp-mark"), + Type: firewall.ChainTypeFilter, + Policy: ptr.To(firewall.ChainPolicyAccept), + Hook: ptr.To(firewall.ChainHookForward), + Priority: ptr.To(firewall.ChainPriorityFilter), + Rules: firewall.RulesSet{ + FilterRules: forwardRules, + }, + }, + { + Name: ptr.To("ecmp-restore-mark"), + Type: firewall.ChainTypeFilter, + Policy: ptr.To(firewall.ChainPolicyAccept), + Hook: ptr.To(firewall.ChainHookPrerouting), + Priority: ptr.To(firewall.ChainPriorityFilter), + Rules: firewall.RulesSet{ + FilterRules: preroutingRules, + }, + }, + }, nil +} + +// sanitizeCIDR returns a string that can be used inside a Kubernetes resource name. +func sanitizeCIDR(cidr networkingv1beta1.CIDR) string { + return strings.NewReplacer(".", "-", "/", "-").Replace(string(cidr)) +} diff --git a/pkg/liqo-controller-manager/networking/internal-network/internalfabric-controller/internalfabric_controller.go b/pkg/liqo-controller-manager/networking/internal-network/internalfabric-controller/internalfabric_controller.go index ab660bb568..d9a5691fc6 100644 --- a/pkg/liqo-controller-manager/networking/internal-network/internalfabric-controller/internalfabric_controller.go +++ b/pkg/liqo-controller-manager/networking/internal-network/internalfabric-controller/internalfabric_controller.go @@ -15,20 +15,31 @@ package internalfabriccontroller import ( + "cmp" "context" "fmt" + "slices" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" networkingv1beta1 "github.com/liqotech/liqo/apis/networking/v1beta1" "github.com/liqotech/liqo/pkg/consts" + "github.com/liqotech/liqo/pkg/gateway" + "github.com/liqotech/liqo/pkg/gateway/forge" + "github.com/liqotech/liqo/pkg/liqo-controller-manager/networking/internal-network/id" ) // InternalFabricReconciler manage InternalFabric lifecycle. @@ -57,6 +68,10 @@ func NewInternalFabricReconciler(cl client.Client, s *runtime.Scheme, routeConfi // +kubebuilder:rbac:groups=networking.liqo.io,resources=genevetunnels/finalizers,verbs=update // +kubebuilder:rbac:groups=networking.liqo.io,resources=internalnodes,verbs=get;list;watch // +kubebuilder:rbac:groups=networking.liqo.io,resources=internalnodes/finalizers,verbs=update +// +kubebuilder:rbac:groups=networking.liqo.io,resources=connections,verbs=get;list;watch +// +kubebuilder:rbac:groups=networking.liqo.io,resources=firewallconfigurations,verbs=get;list;watch;delete;create;update;patch +// +kubebuilder:rbac:groups=networking.liqo.io,resources=firewallconfigurations/finalizers,verbs=update +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch // Reconcile manage InternalFabric lifecycle. func (r *InternalFabricReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -87,12 +102,49 @@ func (r *InternalFabricReconciler) Reconcile(ctx context.Context, req ctrl.Reque return ctrl.Result{}, nil } + // Ensure stable ECMP marks are allocated for this fabric before building route/firewall + // configurations, so that mark values never change on reconnect. + if err := r.ensureECMPMark(ctx, internalFabric); err != nil { + return ctrl.Result{}, fmt.Errorf("ensuring ECMP mark: %w", err) + } + + // List all sibling InternalFabrics with the same remote-cluster-id to build ECMP routes. + siblings, err := r.listSiblingInternalFabrics(ctx, internalFabric) + if err != nil { + return ctrl.Result{}, fmt.Errorf("listing sibling InternalFabrics: %w", err) + } + // route configuration - if err := r.ensureRouteConfiguration(ctx, internalFabric); err != nil { + connected, currentConnected, err := r.connectedGateways(ctx, internalFabric, siblings) + if err != nil { + return ctrl.Result{}, fmt.Errorf("filtering connected gateways: %w", err) + } + + if err := r.ensureRouteConfiguration(ctx, internalFabric, siblings, connected); err != nil { return ctrl.Result{}, fmt.Errorf("ensuring route configuration: %w", err) } + // Per-gateway return-path route configuration: + // - Only needed when ECMP is active (multiple connected gateways). + // - Must be removed for disconnected gateways so that restored conntrack marks + // do not keep steering traffic to a dead tunnel. + if len(connected) > 1 && currentConnected { + if err := r.ensurePerGatewayReturnRouteConfiguration(ctx, internalFabric); err != nil { + return ctrl.Result{}, fmt.Errorf("ensuring per-gateway return route configuration: %w", err) + } + } else { + if err := r.cleanupPerGatewayReturnRouteConfiguration(ctx, internalFabric); err != nil { + return ctrl.Result{}, fmt.Errorf("cleaning up per-gateway return route configuration: %w", err) + } + } + + // firewall configuration (ECMP connection tracking marks) + + if err := r.ensureFirewallConfiguration(ctx, internalFabric, siblings); err != nil { + return ctrl.Result{}, fmt.Errorf("ensuring firewall configuration: %w", err) + } + // geneve tunnels var internalNodeList networkingv1beta1.InternalNodeList @@ -111,6 +163,31 @@ func (r *InternalFabricReconciler) Reconcile(ctx context.Context, req ctrl.Reque return ctrl.Result{}, nil } +// ensureECMPMark ensures that the reconciled InternalFabric has a stable ECMP policy-routing +// mark allocated and persisted in its status. It must be called before ensureRouteConfiguration +// and ensureFirewallConfiguration, which rely on mark values being present. +// Sibling marks are allocated by their own reconciliations; route/firewall functions will return +// an error (and requeue) if a sibling mark is not yet present. +func (r *InternalFabricReconciler) ensureECMPMark(ctx context.Context, internalFabric *networkingv1beta1.InternalFabric) error { + if internalFabric.Status.ECMPMark != nil { + return nil + } + + manager := id.GetECMPMarkManager(ctx, r.Client) + mark, err := manager.Allocate(client.ObjectKeyFromObject(internalFabric).String()) + if err != nil { + return fmt.Errorf("allocating ECMP mark for InternalFabric %q: %w", + client.ObjectKeyFromObject(internalFabric), err) + } + internalFabric.Status.ECMPMark = ptr.To(int(mark)) + if err := r.Status().Update(ctx, internalFabric); err != nil { + return fmt.Errorf("updating ECMP mark status for InternalFabric %q: %w", + client.ObjectKeyFromObject(internalFabric), err) + } + + return nil +} + // SetupWithManager register the InternalFabricReconciler to the manager. func (r *InternalFabricReconciler) SetupWithManager(mgr ctrl.Manager) error { internalNodeEnqueuer := handler.EnqueueRequestsFromMapFunc( @@ -135,10 +212,213 @@ func (r *InternalFabricReconciler) SetupWithManager(mgr ctrl.Manager) error { }, ) + connectionEnqueuer := handler.EnqueueRequestsFromMapFunc( + func(ctx context.Context, obj client.Object) []reconcile.Request { + conn, ok := obj.(*networkingv1beta1.Connection) + if !ok { + return nil + } + + gatewayNamespace := conn.Spec.GatewayRef.Namespace + if gatewayNamespace == "" { + gatewayNamespace = conn.Namespace + } + + internalFabric := &networkingv1beta1.InternalFabric{} + if err := r.Get(ctx, types.NamespacedName{Namespace: gatewayNamespace, Name: conn.Spec.GatewayRef.Name}, internalFabric); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + klog.Errorf("getting the InternalFabric %q/%q: %s", gatewayNamespace, conn.Spec.GatewayRef.Name, err) + return nil + } + + return []reconcile.Request{ + { + NamespacedName: client.ObjectKeyFromObject(internalFabric), + }, + } + }, + ) + + gatewayPodEnqueuer := handler.EnqueueRequestsFromMapFunc( + func(ctx context.Context, obj client.Object) []reconcile.Request { + pod, ok := obj.(*corev1.Pod) + if !ok { + return nil + } + + gwName, ok := pod.Labels[consts.GatewayNameLabel] + if !ok { + return nil + } + + internalFabric := &networkingv1beta1.InternalFabric{} + if err := r.Get(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: gwName}, internalFabric); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + klog.Errorf("Unable to get InternalFabric %q/%q: %s", pod.Namespace, gwName, err) + return nil + } + + return []reconcile.Request{ + { + NamespacedName: client.ObjectKeyFromObject(internalFabric), + }, + } + }, + ) + + gatewayPodPredicate := predicate.NewPredicateFuncs(func(obj client.Object) bool { + labels := obj.GetLabels() + return labels[consts.NetworkingComponentKey] == gateway.GatewayComponentGateway && labels[consts.GatewayNameLabel] != "" + }) + return ctrl.NewControllerManagedBy(mgr).Named(consts.CtrlInternalFabricCM). For(&networkingv1beta1.InternalFabric{}). Watches(&networkingv1beta1.InternalNode{}, internalNodeEnqueuer). + Watches(&networkingv1beta1.Connection{}, connectionEnqueuer). + Watches(&corev1.Pod{}, gatewayPodEnqueuer, builder.WithPredicates(gatewayPodPredicate)). Owns(&networkingv1beta1.RouteConfiguration{}). Owns(&networkingv1beta1.GeneveTunnel{}). Complete(r) } + +// listSiblingInternalFabrics returns all InternalFabrics with the same remote-cluster-id label, +// including the one being reconciled. +func (r *InternalFabricReconciler) listSiblingInternalFabrics(ctx context.Context, + internalFabric *networkingv1beta1.InternalFabric) ([]networkingv1beta1.InternalFabric, error) { + remoteClusterID, ok := internalFabric.Labels[consts.RemoteClusterID] + if !ok { + return nil, fmt.Errorf("InternalFabric %q has no %q label", client.ObjectKeyFromObject(internalFabric), consts.RemoteClusterID) + } + + var list networkingv1beta1.InternalFabricList + if err := r.List(ctx, &list, client.InNamespace(internalFabric.Namespace), client.MatchingLabels{ + consts.RemoteClusterID: remoteClusterID, + }); err != nil { + return nil, err + } + + result := make([]networkingv1beta1.InternalFabric, 0, len(list.Items)) + for i := range list.Items { + if list.Items[i].DeletionTimestamp.IsZero() && list.Items[i].Spec.Interface.Node.Name != "" { + result = append(result, list.Items[i]) + } + } + + // Sort the result by gateway name so that the order is stable across reconciliations. + slices.SortFunc(result, func(a, b networkingv1beta1.InternalFabric) int { + return cmp.Compare(gatewayNameFromInternalFabric(&a), gatewayNameFromInternalFabric(&b)) + }) + + return result, nil +} + +// connectedGateways returns the subset of sibling InternalFabrics whose related Connection resource is in Connected state, +// and whose gateway pod is currently running, along with a boolean indicating whether the provided InternalFabric itself is connected. +func (r *InternalFabricReconciler) connectedGateways(ctx context.Context, + internalFabric *networkingv1beta1.InternalFabric, fabrics []networkingv1beta1.InternalFabric) ([]networkingv1beta1.InternalFabric, bool, error) { + if _, ok := gatewayNameFromOwnerRef(internalFabric); !ok { + klog.V(4).Infof("InternalFabric %q has no gateway owner reference, cannot filter by Connection state", + client.ObjectKeyFromObject(internalFabric)) + return fabrics, true, nil + } + + connected := make([]networkingv1beta1.InternalFabric, 0, len(fabrics)) + currentConnected := false + currentName := internalFabric.Name + for i := range fabrics { + fabric := &fabrics[i] + gwName := gatewayNameFromInternalFabric(fabric) + + connectionName := forge.GatewayResourceName(gwName) + connection := &networkingv1beta1.Connection{} + if err := r.Get(ctx, types.NamespacedName{ + Namespace: internalFabric.Namespace, + Name: connectionName, + }, connection); err != nil { + if apierrors.IsNotFound(err) { + klog.V(4).Infof("Connection %q/%q not found, skipping fabric %q", + internalFabric.Namespace, connectionName, gwName) + continue + } + return nil, false, fmt.Errorf("getting Connection %q/%q: %w", + internalFabric.Namespace, connectionName, err) + } + + if connection.Status.Value != networkingv1beta1.Connected { + klog.V(4).Infof("Connection %q/%q status is %q, skipping fabric %q", + internalFabric.Namespace, connectionName, connection.Status.Value, gwName) + continue + } + + running, err := r.gatewayPodRunning(ctx, fabric) + if err != nil { + return nil, false, fmt.Errorf("checking gateway pod for fabric %q: %w", gwName, err) + } + + if !running { + klog.V(4).Infof("Gateway pod for fabric %q is not running, skipping fabric", gwName) + continue + } + + connected = append(connected, *fabric) + if fabric.Name == currentName { + currentConnected = true + } + } + + return connected, currentConnected, nil +} + +// gatewayPodRunning returns true if there is at least one running gateway pod for the given InternalFabric. +func (r *InternalFabricReconciler) gatewayPodRunning(ctx context.Context, + internalFabric *networkingv1beta1.InternalFabric) (bool, error) { + gwName := gatewayNameFromInternalFabric(internalFabric) + + var podList corev1.PodList + if err := r.List(ctx, &podList, client.InNamespace(internalFabric.Namespace), client.MatchingLabels{ + consts.NetworkingComponentKey: gateway.GatewayComponentGateway, + consts.GatewayNameLabel: gwName, + }); err != nil { + return false, fmt.Errorf("listing gateway pods for %q: %w", gwName, err) + } + + for i := range podList.Items { + pod := &podList.Items[i] + if pod.DeletionTimestamp.IsZero() && pod.Status.Phase == corev1.PodRunning && pod.Status.PodIP != "" { + return true, nil + } + } + + return false, nil +} + +// gatewayNameFromInternalFabric returns the gateway name from the InternalFabric's controller owner reference, +// falling back to the InternalFabric name. +func gatewayNameFromInternalFabric(internalFabric *networkingv1beta1.InternalFabric) string { + if owner := metav1.GetControllerOf(internalFabric); owner != nil { + return owner.Name + } + return internalFabric.Name +} + +// gatewayNameFromOwnerRef returns the name of the gateway owner reference of the InternalFabric. +func gatewayNameFromOwnerRef(internalFabric *networkingv1beta1.InternalFabric) (string, bool) { + owner := metav1.GetControllerOf(internalFabric) + if owner == nil { + return "", false + } + return owner.Name, true +} + +// fabricECMPMark returns the stable ECMP policy-routing mark assigned to the given InternalFabric. +// It returns an error if the mark has not been allocated yet (ensureECMPMarks should be called first). +func fabricECMPMark(internalFabric *networkingv1beta1.InternalFabric) (int, error) { + if internalFabric.Status.ECMPMark == nil { + return 0, fmt.Errorf("InternalFabric %q has no ECMP mark allocated", client.ObjectKeyFromObject(internalFabric)) + } + return *internalFabric.Status.ECMPMark, nil +} diff --git a/pkg/liqo-controller-manager/networking/internal-network/internalfabric-controller/internalfabric_controller_test.go b/pkg/liqo-controller-manager/networking/internal-network/internalfabric-controller/internalfabric_controller_test.go new file mode 100644 index 0000000000..fbd43f7f8c --- /dev/null +++ b/pkg/liqo-controller-manager/networking/internal-network/internalfabric-controller/internalfabric_controller_test.go @@ -0,0 +1,152 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internalfabriccontroller + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + networkingv1beta1 "github.com/liqotech/liqo/apis/networking/v1beta1" + "github.com/liqotech/liqo/pkg/consts" + "github.com/liqotech/liqo/pkg/liqo-controller-manager/networking/internal-network/id" +) + +var _ = Describe("InternalFabric Controller Tests", func() { + + var ( + ctx context.Context + scheme *runtime.Scheme + cl client.Client + rec *InternalFabricReconciler + ) + + BeforeEach(func() { + ctx = context.Background() + scheme = runtime.NewScheme() + Expect(networkingv1beta1.SchemeBuilder.AddToScheme(scheme)).To(Succeed()) + + // Reset the ECMP mark singleton so tests are isolated. + id.ResetECMPMarkManager() + }) + + Context("ECMP mark allocation", func() { + + It("should allocate a mark for an InternalFabric without one", func() { + fabric := &networkingv1beta1.InternalFabric{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fabric", + Namespace: corev1.NamespaceDefault, + Labels: map[string]string{ + consts.RemoteClusterID: "cluster-a", + }, + }, + Spec: networkingv1beta1.InternalFabricSpec{ + Interface: networkingv1beta1.InternalFabricSpecInterface{ + Node: networkingv1beta1.InternalFabricSpecInterfaceNode{Name: "liqo-node"}, + Gateway: networkingv1beta1.InternalFabricSpecInterfaceGateway{ + IP: networkingv1beta1.IP("10.0.0.1"), + }, + }, + }, + } + cl = fake.NewClientBuilder().WithScheme(scheme).WithObjects(fabric).WithStatusSubresource(fabric).Build() + rec = &InternalFabricReconciler{Client: cl, Scheme: scheme} + + Expect(rec.ensureECMPMark(ctx, fabric)).To(Succeed()) + Expect(fabric.Status.ECMPMark).To(Not(BeNil())) + Expect(*fabric.Status.ECMPMark).To(BeNumerically(">=", consts.ECMPReplicaMarkBase)) + }) + + It("should not allocate marks for siblings", func() { + fabric1 := &networkingv1beta1.InternalFabric{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fabric-1", + Namespace: corev1.NamespaceDefault, + Labels: map[string]string{ + consts.RemoteClusterID: "cluster-a", + }, + }, + Spec: networkingv1beta1.InternalFabricSpec{ + Interface: networkingv1beta1.InternalFabricSpecInterface{ + Node: networkingv1beta1.InternalFabricSpecInterfaceNode{Name: "liqo-node"}, + Gateway: networkingv1beta1.InternalFabricSpecInterfaceGateway{ + IP: networkingv1beta1.IP("10.0.0.1"), + }, + }, + }, + } + fabric2 := &networkingv1beta1.InternalFabric{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fabric-2", + Namespace: corev1.NamespaceDefault, + Labels: map[string]string{ + consts.RemoteClusterID: "cluster-a", + }, + }, + Spec: networkingv1beta1.InternalFabricSpec{ + Interface: networkingv1beta1.InternalFabricSpecInterface{ + Node: networkingv1beta1.InternalFabricSpecInterfaceNode{Name: "liqo-node-2"}, + Gateway: networkingv1beta1.InternalFabricSpecInterfaceGateway{ + IP: networkingv1beta1.IP("10.0.0.2"), + }, + }, + }, + } + cl = fake.NewClientBuilder().WithScheme(scheme).WithObjects(fabric1, fabric2). + WithStatusSubresource(fabric1, fabric2).Build() + rec = &InternalFabricReconciler{Client: cl, Scheme: scheme} + + Expect(rec.ensureECMPMark(ctx, fabric1)).To(Succeed()) + Expect(fabric1.Status.ECMPMark).To(Not(BeNil())) + Expect(fabric2.Status.ECMPMark).To(BeNil()) + }) + + It("should keep an existing mark stable", func() { + existingMark := consts.ECMPReplicaMarkBase + 7 + fabric := &networkingv1beta1.InternalFabric{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fabric", + Namespace: corev1.NamespaceDefault, + Labels: map[string]string{ + consts.RemoteClusterID: "cluster-a", + }, + }, + Spec: networkingv1beta1.InternalFabricSpec{ + Interface: networkingv1beta1.InternalFabricSpecInterface{ + Node: networkingv1beta1.InternalFabricSpecInterfaceNode{Name: "liqo-node"}, + Gateway: networkingv1beta1.InternalFabricSpecInterfaceGateway{ + IP: networkingv1beta1.IP("10.0.0.1"), + }, + }, + }, + Status: networkingv1beta1.InternalFabricStatus{ + ECMPMark: &existingMark, + }, + } + cl = fake.NewClientBuilder().WithScheme(scheme).WithObjects(fabric).WithStatusSubresource(fabric).Build() + rec = &InternalFabricReconciler{Client: cl, Scheme: scheme} + + Expect(rec.ensureECMPMark(ctx, fabric)).To(Succeed()) + Expect(*fabric.Status.ECMPMark).To(Equal(existingMark)) + }) + }) +}) diff --git a/pkg/liqo-controller-manager/networking/internal-network/internalfabric-controller/route.go b/pkg/liqo-controller-manager/networking/internal-network/internalfabric-controller/route.go index bee522ed6e..0864f12ac8 100644 --- a/pkg/liqo-controller-manager/networking/internal-network/internalfabric-controller/route.go +++ b/pkg/liqo-controller-manager/networking/internal-network/internalfabric-controller/route.go @@ -17,7 +17,7 @@ package internalfabriccontroller import ( "context" "fmt" - "sort" + "slices" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -27,18 +27,35 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" networkingv1beta1 "github.com/liqotech/liqo/apis/networking/v1beta1" + "github.com/liqotech/liqo/pkg/consts" "github.com/liqotech/liqo/pkg/fabric" "github.com/liqotech/liqo/pkg/utils/resource" ) -func (r *InternalFabricReconciler) ensureRouteConfiguration(ctx context.Context, internalFabric *networkingv1beta1.InternalFabric) error { +const defaultMarkRoutePriority = 10001 +const defaultECMPRoutePriority = 10000 + +// GenerateRouteConfigurationName returns the name of the RouteConfiguration associated to the InternalFabric. +// It uses the remote-cluster-id from the InternalFabric labels so all siblings share the same name. +func generateRouteConfigurationName(remoteClusterID string) string { + return fmt.Sprintf("%s-node-gw", remoteClusterID) +} + +func (r *InternalFabricReconciler) ensureRouteConfiguration(ctx context.Context, + internalFabric *networkingv1beta1.InternalFabric, siblings []networkingv1beta1.InternalFabric, + connected []networkingv1beta1.InternalFabric) error { if internalFabric.Spec.Interface.Node.Name == "" { return fmt.Errorf("internal fabric %q has node interface name empty", client.ObjectKeyFromObject(internalFabric)) } + remoteClusterID, ok := internalFabric.Labels[consts.RemoteClusterID] + if !ok || remoteClusterID == "" { + return fmt.Errorf("InternalFabric %q does not have a remote-cluster-id label", internalFabric.Name) + } + route := &networkingv1beta1.RouteConfiguration{ ObjectMeta: metav1.ObjectMeta{ - Name: GenerateRouteConfigurationName(internalFabric), + Name: generateRouteConfigurationName(remoteClusterID), Namespace: internalFabric.Namespace, }, } @@ -49,43 +66,132 @@ func (r *InternalFabricReconciler) ensureRouteConfiguration(ctx context.Context, } route.SetLabels(labels.Merge(route.Labels, fabric.ForgeRouteTargetLabels())) - // Add route rule for every remote CIDR var rules []networkingv1beta1.Rule - var priority *int + priority := ptr.To(defaultMarkRoutePriority) if r.RouteConfigurationRulePriority > 0 { priority = ptr.To(r.RouteConfigurationRulePriority) } - rules = append(rules, networkingv1beta1.Rule{ - Dst: ptr.To(networkingv1beta1.CIDR(fmt.Sprintf("%s/32", internalFabric.Spec.Interface.Gateway.IP))), - Priority: priority, - Routes: []networkingv1beta1.Route{ - { - Dst: ptr.To(networkingv1beta1.CIDR(fmt.Sprintf("%s/32", internalFabric.Spec.Interface.Gateway.IP))), - Dev: ptr.To(internalFabric.Spec.Interface.Node.Name), - Scope: ptr.To(networkingv1beta1.LinkScope), + // Add a link-scope route for each configured gateway's inner IP. + for i := range siblings { + fabric := &siblings[i] + rules = append(rules, networkingv1beta1.Rule{ + Dst: ptr.To(networkingv1beta1.CIDR(fmt.Sprintf("%s/32", fabric.Spec.Interface.Gateway.IP))), + Priority: priority, + Routes: []networkingv1beta1.Route{ + { + Dst: ptr.To(networkingv1beta1.CIDR(fmt.Sprintf("%s/32", fabric.Spec.Interface.Gateway.IP))), + Dev: ptr.To(fabric.Spec.Interface.Node.Name), + Scope: ptr.To(networkingv1beta1.LinkScope), + }, }, - }, - }) + }) + } - remoteCIDRs := internalFabric.Spec.RemoteCIDRs + remoteCIDRs := slices.Clone(internalFabric.Spec.RemoteCIDRs) // clone to avoid modifying the original slice and invalidating the cache // sort slice to prevent useless updates if CIDRs are in different order - sort.Slice(remoteCIDRs, func(i, j int) bool { - return remoteCIDRs[i] < remoteCIDRs[j] - }) + slices.Sort(remoteCIDRs) + + // Add one rule per remote CIDR. + // Use a classic single gateway when there is only one connected gateway, otherwise use ECMP next-hops. + for _, remoteCIDR := range remoteCIDRs { + var route networkingv1beta1.Route + route.Dst = ptr.To(remoteCIDR) + + switch len(connected) { + case 0: + continue + case 1: + route.Gw = ptr.To(connected[0].Spec.Interface.Gateway.IP) + default: + nextHops := make([]networkingv1beta1.NextHop, 0, len(connected)) + for i := range connected { + fab := &connected[i] + nextHops = append(nextHops, networkingv1beta1.NextHop{ + Gw: fab.Spec.Interface.Gateway.IP, + Dev: fab.Spec.Interface.Node.Name, + }) + } + route.NextHops = nextHops + } + + rules = append(rules, networkingv1beta1.Rule{ + Routes: []networkingv1beta1.Route{route}, + Dst: ptr.To(remoteCIDR), + Priority: priority, + }) + } + + route.Spec = networkingv1beta1.RouteConfigurationSpec{ + Table: networkingv1beta1.Table{ + Name: route.Name, + Rules: rules, + }, + } + + return controllerutil.SetOwnerReference(internalFabric, route, r.Scheme) + }) + if err != nil { + klog.Errorf("Unable to create or update RouteConfiguration %q: %s", route.Name, err) + return err + } + + return nil +} + +func (r *InternalFabricReconciler) ensurePerGatewayReturnRouteConfiguration(ctx context.Context, + internalFabric *networkingv1beta1.InternalFabric) error { + remoteCIDRs := slices.Clone(internalFabric.Spec.RemoteCIDRs) // clone to avoid modifying the original slice and invalidating the cache + slices.Sort(remoteCIDRs) + + markPriority := ptr.To(defaultECMPRoutePriority) + if r.RouteConfigurationRulePriority > 0 { + // We make sure marks always have priority slightly lower than the main route configuration rules. + // to avoid to do ECMP on return traffic. + markPriority = ptr.To(r.RouteConfigurationRulePriority - 1) + } + + remoteClusterID, ok := internalFabric.Labels[consts.RemoteClusterID] + if !ok { + return fmt.Errorf("InternalFabric %q has no %q label", client.ObjectKeyFromObject(internalFabric), consts.RemoteClusterID) + } + + gwName := gatewayNameFromInternalFabric(internalFabric) + name := replicaRouteConfigurationName(remoteClusterID, gwName) + + mark, err := fabricECMPMark(internalFabric) + if err != nil { + return fmt.Errorf("ensuring route configuration: %w", err) + } + + route := &networkingv1beta1.RouteConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: internalFabric.Namespace, + }, + } + _, err = resource.CreateOrUpdate(ctx, r.Client, route, func() error { + if route.Labels == nil { + route.Labels = make(labels.Set) + } + route.SetLabels(labels.Merge(route.Labels, fabric.ForgeRouteTargetLabels())) + route.Labels[consts.InternalFabricName] = internalFabric.Name + + var rules []networkingv1beta1.Rule for _, remoteCIDR := range remoteCIDRs { - rule := networkingv1beta1.Rule{ + rules = append(rules, networkingv1beta1.Rule{ + FwMark: ptr.To(mark), + Dst: ptr.To(remoteCIDR), + Priority: markPriority, Routes: []networkingv1beta1.Route{ { Dst: ptr.To(remoteCIDR), Gw: ptr.To(internalFabric.Spec.Interface.Gateway.IP), + Dev: ptr.To(internalFabric.Spec.Interface.Node.Name), }, }, - Dst: ptr.To(remoteCIDR), - Priority: priority, - } - rules = append(rules, rule) + }) } route.Spec = networkingv1beta1.RouteConfigurationSpec{ @@ -97,15 +203,41 @@ func (r *InternalFabricReconciler) ensureRouteConfiguration(ctx context.Context, return controllerutil.SetControllerReference(internalFabric, route, r.Scheme) }) + if err != nil { - klog.Errorf("Unable to create or update RouteConfiguration %q: %s", route.Name, err) + klog.Errorf("Unable to create or update RouteConfiguration %q: %s", name, err) return err } return nil } -// GenerateRouteConfigurationName returns the name of the RouteConfiguration associated to the InternalFabric. -func GenerateRouteConfigurationName(internalFabric *networkingv1beta1.InternalFabric) string { - return fmt.Sprintf("%s-node-gw", internalFabric.Name) +func (r *InternalFabricReconciler) cleanupPerGatewayReturnRouteConfiguration(ctx context.Context, + internalFabric *networkingv1beta1.InternalFabric) error { + remoteClusterID, ok := internalFabric.Labels[consts.RemoteClusterID] + if !ok { + return nil + } + + gwName := gatewayNameFromInternalFabric(internalFabric) + name := replicaRouteConfigurationName(remoteClusterID, gwName) + + route := &networkingv1beta1.RouteConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: internalFabric.Namespace, + }, + } + err := r.Delete(ctx, route) + if client.IgnoreNotFound(err) != nil { + return fmt.Errorf("deleting per-gateway return path RouteConfiguration %q: %w", name, err) + } + return nil +} + +// replicaRouteConfigurationName returns the name of the per-gateway RouteConfiguration. +// The name is unique per gateway within a remote cluster, not per owner fabric, so that all sibling fabrics +// reconcile the same replica table for a given gateway. +func replicaRouteConfigurationName(remoteClusterID, gatewayName string) string { + return fmt.Sprintf("%s-node-gw-replica-%s", remoteClusterID, gatewayName) } diff --git a/pkg/liqo-controller-manager/networking/internal-network/server-controller/server_controller.go b/pkg/liqo-controller-manager/networking/internal-network/server-controller/server_controller.go index 8443df5fd2..d6e5f195da 100644 --- a/pkg/liqo-controller-manager/networking/internal-network/server-controller/server_controller.go +++ b/pkg/liqo-controller-manager/networking/internal-network/server-controller/server_controller.go @@ -110,14 +110,9 @@ func (r *ServerReconciler) ensureInternalFabric(ctx context.Context, gwServer *n return fmt.Errorf("internal endpoint not found for the gateway server %q", gwServer.Name) } - internalFabricName, err := netutils.ForgeInternalFabricName(ctx, r.Client, &gwServer.ObjectMeta) - if err != nil { - return fmt.Errorf("unable to retrieve the cluster ID from the gateway server %q", gwServer.Name) - } - internalFabric := &networkingv1beta1.InternalFabric{ ObjectMeta: metav1.ObjectMeta{ - Name: internalFabricName, + Name: gwServer.Name, Namespace: gwServer.Namespace, }, } @@ -176,15 +171,18 @@ func (r *ServerReconciler) gatewayServerEnqueuerByRemoteID() handler.MapFunc { return nil } - gwServer, err := getters.GetGatewayServerByClusterID(ctx, r.Client, remoteClusterID, corev1.NamespaceAll) + gwServerList, err := getters.ListGatewayServersByClusterID(ctx, r.Client, remoteClusterID, corev1.NamespaceAll) if err != nil { - if apierrors.IsNotFound(err) { - return nil - } - klog.Errorf("unable to get the gateway server for cluster %s: %s", remoteClusterID, err) + klog.Errorf("unable to list gateway servers for cluster %s: %s", remoteClusterID, err) return nil } - return []reconcile.Request{{NamespacedName: client.ObjectKeyFromObject(gwServer)}} + var requests []reconcile.Request + for i := range gwServerList.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(&gwServerList.Items[i]), + }) + } + return requests } } diff --git a/pkg/liqo-controller-manager/networking/utils/internalfabric.go b/pkg/liqo-controller-manager/networking/utils/internalfabric.go deleted file mode 100644 index 6c4f31317e..0000000000 --- a/pkg/liqo-controller-manager/networking/utils/internalfabric.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2019-2026 The Liqo Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package utils //nolint:revive // we want to use this package name - -import ( - "context" - "fmt" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - networkingv1beta1 "github.com/liqotech/liqo/apis/networking/v1beta1" - "github.com/liqotech/liqo/pkg/utils/getters" -) - -// ForgeInternalFabricName retrieves the name of the InternalFabric resource associated with the given metadata. -// WARNING: this function try to check if an InternalFabric resource already -// exists with the name of the gateway resource. -// This is intended to avoid breaking changes, since the InternalFabric name -// has changed from Gateway's name to the Gateway's remote cluster ID. -func ForgeInternalFabricName(ctx context.Context, cl client.Client, meta *metav1.ObjectMeta) (string, error) { - internalFabric := &networkingv1beta1.InternalFabric{} - - err := cl.Get(ctx, client.ObjectKey{ - Name: meta.Name, - Namespace: meta.Namespace, - }, internalFabric) - - switch { - case apierrors.IsNotFound(err): - remoteClusterID, err := getters.RetrieveRemoteClusterIDFromMeta(meta) - if err != nil { - return "", fmt.Errorf("unable to retrieve remote cluster ID from object %s/%s:%w", - meta.GetNamespace(), meta.GetName(), err) - } - return string(remoteClusterID), nil - case err != nil: - return "", fmt.Errorf("unable to get the internal fabric %q: %w", meta.Name, err) - } - return internalFabric.Name, nil -} diff --git a/pkg/liqo-controller-manager/offloading/storageprovisioner/remoteprovisioner.go b/pkg/liqo-controller-manager/offloading/storageprovisioner/remoteprovisioner.go index b53f4c322c..84cc29492d 100644 --- a/pkg/liqo-controller-manager/offloading/storageprovisioner/remoteprovisioner.go +++ b/pkg/liqo-controller-manager/offloading/storageprovisioner/remoteprovisioner.go @@ -35,21 +35,12 @@ import ( // ProvisionRemotePVC ensures the existence of a remote PVC and returns a virtual PV for that remote storage device. func ProvisionRemotePVC(ctx context.Context, options controller.ProvisionOptions, - remoteNamespace, remoteRealStorageClass string, + remoteNamespace, localVirtualStorageClass, remoteRealStorageClass string, remotePvcLister corev1listers.PersistentVolumeClaimNamespaceLister, remotePvcClient corev1clients.PersistentVolumeClaimInterface, forgingOpts *forge.ForgingOpts) (*corev1.PersistentVolume, controller.ProvisioningState, error) { virtualPvc := options.PVC - labels := options.SelectedNode.GetLabels() - if labels == nil { - return nil, controller.ProvisioningInBackground, fmt.Errorf("no labels found for node %s", options.SelectedNode.GetName()) - } - remoteClusterID, ok := labels[consts.RemoteClusterID] - if !ok { - return nil, controller.ProvisioningInBackground, fmt.Errorf("no remote cluster ID found for node %s", options.SelectedNode.GetName()) - } - // get the storage class for the remote PVC, // use its class if denied, otherwise use the default one var remoteStorageClass string @@ -57,6 +48,10 @@ func ProvisionRemotePVC(ctx context.Context, switch { case apierrors.IsNotFound(err): remoteStorageClass = remoteRealStorageClass + // if the local storage class is an OMNI one, we need to remove the prefix to get the real remote storage class + if strings.HasPrefix(localVirtualStorageClass, consts.OmniStorageClassPrefix) { + remoteStorageClass = strings.TrimPrefix(localVirtualStorageClass, consts.OmniStorageClassPrefix) + } case err != nil && !apierrors.IsNotFound(err): return nil, controller.ProvisioningInBackground, err case remotePvc.Spec.StorageClassName != nil: @@ -78,11 +73,17 @@ func ProvisionRemotePVC(ctx context.Context, } mutation := remotePersistentVolumeClaim(virtualPvc, remoteStorageClass, remoteNamespace, forgingOpts, remoteAccessModes) + _, err = remotePvcClient.Apply(ctx, mutation, forge.ApplyOptions()) if err != nil { return nil, controller.ProvisioningInBackground, err } + nodeAffinity, err := buildPvNodeAffinity(options) + if err != nil { + return nil, controller.ProvisioningInBackground, err + } + pv := &corev1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: options.PVName, @@ -98,21 +99,7 @@ func ProvisionRemotePVC(ctx context.Context, Path: "/tmp/liqo-placeholder", }, }, - NodeAffinity: &corev1.VolumeNodeAffinity{ - Required: &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{ - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: consts.RemoteClusterID, - Operator: corev1.NodeSelectorOpIn, - Values: []string{remoteClusterID}, - }, - }, - }, - }, - }, - }, + NodeAffinity: nodeAffinity, }, } @@ -123,6 +110,49 @@ func ProvisionRemotePVC(ctx context.Context, return pv, controller.ProvisioningFinished, nil } +func buildPvNodeAffinity(options controller.ProvisionOptions) (*corev1.VolumeNodeAffinity, error) { + nodeAffinitySelectorKey := consts.EdgeLocationName + + // Check whether pvc should be provisioned on all the edges, in that case just filter by virtual node. + shouldProvisionOnAllEdges := options.PVC.Annotations != nil && + options.PVC.Annotations[consts.ProvisionPVCOnAllEdgesAnnotationKey] == consts.ProvisionPVCOnAllEdgesAnnotationValue + + nodeAffinityOperator := corev1.NodeSelectorOpIn + var nodeAffinityValues []string + + if shouldProvisionOnAllEdges { + nodeAffinityOperator = corev1.NodeSelectorOpExists + } else { + labels := options.SelectedNode.GetLabels() + if labels == nil { + return nil, fmt.Errorf("no labels found for node %s", options.SelectedNode.GetName()) + } + nodeSelectorValue, ok := labels[nodeAffinitySelectorKey] + if !ok { + return nil, + fmt.Errorf("liqo node selector %q not found on node %s", nodeAffinitySelectorKey, options.SelectedNode.GetName()) + } + nodeAffinityValues = []string{nodeSelectorValue} + } + + res := &corev1.VolumeNodeAffinity{ + Required: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: nodeAffinitySelectorKey, + Operator: nodeAffinityOperator, + Values: nodeAffinityValues, + }, + }, + }, + }, + }, + } + return res, nil +} + // remotePersistentVolumeClaim forges the apply patch for the reflected PersistentVolumeClaim, given the local one. func remotePersistentVolumeClaim(virtualPvc *corev1.PersistentVolumeClaim, storageClass, namespace string, forgingOpts *forge.ForgingOpts, diff --git a/pkg/liqo-controller-manager/offloading/storageprovisioner/storageprovisioner_test.go b/pkg/liqo-controller-manager/offloading/storageprovisioner/storageprovisioner_test.go index 12c04180d7..c455fe5f05 100644 --- a/pkg/liqo-controller-manager/offloading/storageprovisioner/storageprovisioner_test.go +++ b/pkg/liqo-controller-manager/offloading/storageprovisioner/storageprovisioner_test.go @@ -43,7 +43,8 @@ import ( ) const ( - remoteClusterID = "foreign-cluster-id" + remoteClusterID = "foreign-cluster-id" + edgeLocationName = "fast-edge" ) var _ = Describe("Test Storage Provisioner", func() { @@ -73,8 +74,11 @@ var _ = Describe("Test Storage Provisioner", func() { forgeNode = func(name string, isVirtual bool) *corev1.Node { node := &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{liqoconst.RemoteClusterID: remoteClusterID}, + Name: name, + Labels: map[string]string{ + liqoconst.RemoteClusterID: remoteClusterID, + liqoconst.EdgeLocationName: edgeLocationName, + }, }, } @@ -306,7 +310,7 @@ var _ = Describe("Test Storage Provisioner", func() { SelectedNode: &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: virtualNodeName, - Labels: map[string]string{liqoconst.RemoteClusterID: remoteClusterID}, + Labels: map[string]string{liqoconst.EdgeLocationName: edgeLocationName}, }, }, PVC: &corev1.PersistentVolumeClaim{ @@ -335,16 +339,16 @@ var _ = Describe("Test Storage Provisioner", func() { return &policy }(), }, - }, RemoteNamespace, storageClass, remotePersistentVolumeClaims, remotePersistentVolumesClaimsClient, forgingOpts) + }, RemoteNamespace, virtualStorageClassName, storageClass, remotePersistentVolumeClaims, remotePersistentVolumesClaimsClient, forgingOpts) if err != nil { return err } Expect(state).To(Equal(controller.ProvisioningFinished)) Expect(pv).ToNot(BeNil()) - Expect(pv.Spec.NodeAffinity.Required.NodeSelectorTerms[0].MatchExpressions[0].Key).To(Equal(liqoconst.RemoteClusterID)) + Expect(pv.Spec.NodeAffinity.Required.NodeSelectorTerms[0].MatchExpressions[0].Key).To(Equal(liqoconst.EdgeLocationName)) Expect(pv.Spec.NodeAffinity.Required.NodeSelectorTerms[0].MatchExpressions[0].Operator).To(Equal(corev1.NodeSelectorOpIn)) - Expect(pv.Spec.NodeAffinity.Required.NodeSelectorTerms[0].MatchExpressions[0].Values).To(ContainElement(remoteClusterID)) + Expect(pv.Spec.NodeAffinity.Required.NodeSelectorTerms[0].MatchExpressions[0].Values).To(ContainElement(edgeLocationName)) Expect(pv.Spec.StorageClassName).To(Equal(virtualStorageClassName)) _, err = testEnvClient.CoreV1().PersistentVolumeClaims(RemoteNamespace).Get(ctx, pvcName, metav1.GetOptions{}) @@ -373,8 +377,67 @@ var _ = Describe("Test Storage Provisioner", func() { Expect(realPvc.Annotations).ToNot(HaveKey(testutil.FakeNotReflectedAnnotKey)) }) - }) + It("provision remote pvc with provision-on-all-edge-nodes annotation", func() { + pvcWithAnnotation := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName + "-with-annotation", + Namespace: LocalNamespace, + UID: "uuid-with-annotation", + Annotations: map[string]string{ + liqoconst.ProvisionPVCOnAllEdgesAnnotationKey: liqoconst.ProvisionPVCOnAllEdgesAnnotationValue, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: pointer.String(virtualStorageClassName), + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: *resource.NewQuantity(10, resource.BinarySI), + }, + }, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + }, + } + + pv, state, err := ProvisionRemotePVC(ctx, controller.ProvisionOptions{ + SelectedNode: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: virtualNodeName, + Labels: map[string]string{liqoconst.EdgeLocationName: edgeLocationName}, + }, + }, + PVC: pvcWithAnnotation, + PVName: pvName + "-with-annotation", + StorageClass: &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: virtualStorageClassName, + }, + ReclaimPolicy: func() *corev1.PersistentVolumeReclaimPolicy { + policy := corev1.PersistentVolumeReclaimDelete + + return &policy + }(), + }, + }, RemoteNamespace, virtualStorageClassName, realStorageClassName, remotePersistentVolumeClaims, remotePersistentVolumesClaimsClient, forgingOpts) + + Expect(err).ToNot(HaveOccurred()) + Expect(state).To(Equal(controller.ProvisioningFinished)) + Expect(pv).ToNot(BeNil()) + + Expect(pv.Spec.NodeAffinity).ToNot(BeNil()) + Expect(pv.Spec.NodeAffinity.Required).ToNot(BeNil()) + Expect(pv.Spec.NodeAffinity.Required.NodeSelectorTerms).To(HaveLen(1)) + Expect(pv.Spec.NodeAffinity.Required.NodeSelectorTerms[0].MatchExpressions).To(HaveLen(1)) + + matchExpression := pv.Spec.NodeAffinity.Required.NodeSelectorTerms[0].MatchExpressions[0] + Expect(matchExpression.Key).To(Equal(liqoconst.EdgeLocationName)) + Expect(matchExpression.Operator).To(Equal(corev1.NodeSelectorOpExists)) + Expect(matchExpression.Values).To(BeEmpty()) + err = testEnvClient.CoreV1().PersistentVolumeClaims(RemoteNamespace).Delete(ctx, + pvcWithAnnotation.Name, metav1.DeleteOptions{}) + Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) + }) + }) }) Context("utility functions", func() { diff --git a/pkg/liqo-controller-manager/offloading/virtualnode-controller/namespacemap.go b/pkg/liqo-controller-manager/offloading/virtualnode-controller/namespacemap.go index 4d8d1d9550..98ef8a9b23 100644 --- a/pkg/liqo-controller-manager/offloading/virtualnode-controller/namespacemap.go +++ b/pkg/liqo-controller-manager/offloading/virtualnode-controller/namespacemap.go @@ -87,6 +87,12 @@ func (r *VirtualNodeReconciler) ensureNamespaceMapAbsence(ctx context.Context, v for i := range namespaceMapList.Items { nm := &namespaceMapList.Items[i] + // If the NamespaceMap is not managed by Liqo, skip deletion as it is not managed by this controller. + if v, ok := nm.Annotations[liqoconst.ManagedNamespaceMapAnnotKey]; ok && v == liqoconst.ManagedNamespaceMapAnnotValue { + klog.V(4).Infof("NamespaceMap %q is managed by the user, skipping deletion", nm.Name) + continue + } + // Retrieve all the VirtualNodes associated with the NamespaceMap. virtualNodes, err := getters.ListVirtualNodesByClusterID(ctx, r.Client, virtualNodeRemoteClusterID) if err != nil { diff --git a/pkg/liqoctl/info/peer/network.go b/pkg/liqoctl/info/peer/network.go index 0e35a48dc8..80605c799d 100644 --- a/pkg/liqoctl/info/peer/network.go +++ b/pkg/liqoctl/info/peer/network.go @@ -166,21 +166,27 @@ func (nc *NetworkChecker) collectStatusInfo(clusterID liqov1beta1.ClusterID, // collectGatewayInfo collects the info about the local gateway connected to the peer cluster gateway. func (nc *NetworkChecker) collectGatewayInfo(ctx context.Context, cl client.Client, clusterID liqov1beta1.ClusterID, peerNetwork *Network) error { - gwServer, gwClient, err := getters.GetGatewaysByClusterID(ctx, cl, clusterID) + gwServerList, gwClientList, err := getters.GetGatewaysByClusterID(ctx, cl, clusterID) if err != nil { return err } + hasServers := gwServerList != nil && len(gwServerList.Items) > 0 + hasClients := gwClientList != nil && len(gwClientList.Items) > 0 + switch { - case gwClient != nil && gwServer != nil: - return fmt.Errorf("multiple Gateways found") - case gwClient != nil: + case hasClients && hasServers: + return fmt.Errorf("both GatewayServer and GatewayClient found for cluster %s", clusterID) + case hasClients: // We are reflecting the status of the peer cluster. So, if locally there is a client, the peer - // cluester has a server. + // cluster has a server. Use the first client's endpoint. + gwClient := &gwClientList.Items[0] peerNetwork.Gateway.Role = GatewayServerType peerNetwork.Gateway.Address = gwClient.Spec.Endpoint.Addresses peerNetwork.Gateway.Port = gwClient.Spec.Endpoint.Port - case gwServer != nil: + case hasServers: + // Use the first server's endpoint. + gwServer := &gwServerList.Items[0] peerNetwork.Gateway.Role = GatewayClientType if gwServer.Status.Endpoint != nil { peerNetwork.Gateway.Address = gwServer.Status.Endpoint.Addresses diff --git a/pkg/liqoctl/network/cluster.go b/pkg/liqoctl/network/cluster.go index eeccd50446..8e6e47f762 100644 --- a/pkg/liqoctl/network/cluster.go +++ b/pkg/liqoctl/network/cluster.go @@ -396,21 +396,12 @@ func endpointHasChanged(endpoint *networkingv1beta1.Endpoint, service *corev1.Se return false } -// EnsureGatewayServer create or updates a GatewayServer. -func (c *Cluster) EnsureGatewayServer(ctx context.Context, opts *forge.GwServerOptions) (*networkingv1beta1.GatewayServer, error) { - s := c.local.Printer.StartSpinner("Setting up gateway server") - - // Check if the GatewayServer already exists. - var name *string - gwServer, err := getters.GetGatewayServerByClusterID(ctx, c.local.CRClient, c.remoteClusterID, c.localNetworkNamespace) - if client.IgnoreNotFound(err) != nil { - return nil, err - } else if err == nil { - name = &gwServer.Name // if the GatewayServer already exists, keep its name - } +// EnsureGatewayServer creates or updates a single GatewayServer with the given name. +func (c *Cluster) EnsureGatewayServer(ctx context.Context, name string, opts *forge.GwServerOptions) (*networkingv1beta1.GatewayServer, error) { + s := c.local.Printer.StartSpinner(fmt.Sprintf("Setting up gateway server %q", name)) // Forge GatewayServer. - gwServer, err = forge.GatewayServer(c.localNetworkNamespace, name, opts) + gwServer, err := forge.GatewayServer(c.localNetworkNamespace, &name, opts) if err != nil { s.Fail(fmt.Sprintf("An error occurred while forging gateway server: %v", output.PrettyErr(err))) return nil, err @@ -447,20 +438,11 @@ func (c *Cluster) EnsureGatewayServer(ctx context.Context, opts *forge.GwServerO return gwServer, nil } -// EnsureGatewayClient create or updates a GatewayClient. -func (c *Cluster) EnsureGatewayClient(ctx context.Context, opts *forge.GwClientOptions) (*networkingv1beta1.GatewayClient, error) { - s := c.local.Printer.StartSpinner("Setting up gateway client") - - // Check if the GatewayClient already exists. - var name *string - gwClient, err := getters.GetGatewayClientByClusterID(ctx, c.local.CRClient, c.remoteClusterID, c.localNetworkNamespace) - if client.IgnoreNotFound(err) != nil { - return nil, err - } else if err == nil { - name = &gwClient.Name // if the GatewayClient already exists, keep its name - } +// EnsureGatewayClient creates or updates a single GatewayClient with the given name. +func (c *Cluster) EnsureGatewayClient(ctx context.Context, name string, opts *forge.GwClientOptions) (*networkingv1beta1.GatewayClient, error) { + s := c.local.Printer.StartSpinner(fmt.Sprintf("Setting up gateway client %q", name)) - gwClient, err = forge.GatewayClient(c.localNetworkNamespace, name, opts) + gwClient, err := forge.GatewayClient(c.localNetworkNamespace, &name, opts) if err != nil { s.Fail(fmt.Sprintf("An error occurred while forging gateway client: %v", output.PrettyErr(err))) return nil, err @@ -542,86 +524,156 @@ func (c *Cluster) DeleteConfiguration(ctx context.Context, remoteClusterID liqov return nil } -// DeleteGatewayServer deletes a GatewayServer. +// DeleteGatewayServer deletes all GatewayServers for a remote cluster. func (c *Cluster) DeleteGatewayServer(ctx context.Context, remoteClusterID liqov1beta1.ClusterID) error { - s := c.local.Printer.StartSpinner("Deleting gateway server") + s := c.local.Printer.StartSpinner("Deleting gateway server(s)") - // Retrieve GatewayServer. - gwServer, err := getters.GetGatewayServerByClusterID(ctx, c.local.CRClient, remoteClusterID, c.localNetworkNamespace) + gwServerList, err := getters.ListGatewayServersByClusterID(ctx, c.local.CRClient, remoteClusterID, c.localNetworkNamespace) if client.IgnoreNotFound(err) != nil { - s.Fail("An error occurred while retrieving gateway server: ", output.PrettyErr(err)) + s.Fail("An error occurred while listing gateway servers: ", output.PrettyErr(err)) return err - } else if apierrors.IsNotFound(err) { + } + if gwServerList == nil || len(gwServerList.Items) == 0 { s.Success("Gateway server already deleted") return nil } - // Delete GatewayServer. - err = c.local.CRClient.Delete(ctx, gwServer) - switch { - case client.IgnoreNotFound(err) != nil: - s.Fail("An error occurred while deleting gateway server: ", output.PrettyErr(err)) - return err - case apierrors.IsNotFound(err): - s.Success("Gateway server already deleted") - default: - s.Success("Gateway server correctly deleted") + for i := range gwServerList.Items { + gwServer := &gwServerList.Items[i] + if err := c.local.CRClient.Delete(ctx, gwServer); err != nil { + if client.IgnoreNotFound(err) != nil { + s.Fail("An error occurred while deleting gateway server: ", output.PrettyErr(err)) + return err + } + } } + s.Success("Gateway server(s) correctly deleted") return nil } -// DeleteGatewayClient deletes a GatewayClient. +// DeleteGatewayClient deletes all GatewayClients for a remote cluster. func (c *Cluster) DeleteGatewayClient(ctx context.Context, remoteClusterID liqov1beta1.ClusterID) error { - s := c.local.Printer.StartSpinner("Deleting gateway client") + s := c.local.Printer.StartSpinner("Deleting gateway client(s)") - // Retrieve GatewayClient. - gwClient, err := getters.GetGatewayClientByClusterID(ctx, c.local.CRClient, remoteClusterID, c.localNetworkNamespace) + gwClientList, err := getters.ListGatewayClientsByClusterID(ctx, c.local.CRClient, remoteClusterID, c.localNetworkNamespace) if client.IgnoreNotFound(err) != nil { - s.Fail("An error occurred while retrieving gateway client: ", output.PrettyErr(err)) + s.Fail("An error occurred while listing gateway clients: ", output.PrettyErr(err)) return err - } else if apierrors.IsNotFound(err) { + } + if gwClientList == nil || len(gwClientList.Items) == 0 { s.Success("Gateway client already deleted") return nil } - // Delete GatewayClient. - err = c.local.CRClient.Delete(ctx, gwClient) - switch { - case client.IgnoreNotFound(err) != nil: - s.Fail("An error occurred while deleting gateway client: ", output.PrettyErr(err)) - return err - case apierrors.IsNotFound(err): - s.Success("Gateway client already deleted") - default: - s.Success("Gateway client correctly deleted") + for i := range gwClientList.Items { + gwClient := &gwClientList.Items[i] + if err := c.local.CRClient.Delete(ctx, gwClient); err != nil { + if client.IgnoreNotFound(err) != nil { + s.Fail("An error occurred while deleting gateway client: ", output.PrettyErr(err)) + return err + } + } } + s.Success("Gateway client(s) correctly deleted") return nil } -// CheckAlreadyEstablishedForGwServer checks if a GatewayServer is already established. -func (c *Cluster) CheckAlreadyEstablishedForGwServer(ctx context.Context) (bool, error) { - _, err := getters.GetGatewayServerByClusterID(ctx, c.local.CRClient, c.remoteClusterID, c.localNetworkNamespace) - switch { - case client.IgnoreNotFound(err) != nil: +// DeleteStaleGatewayServers deletes GatewayServers for the remote cluster whose names are not in expectedNames. +func (c *Cluster) DeleteStaleGatewayServers(ctx context.Context, expectedNames []string) error { + return c.deleteStaleGateways(ctx, c.remoteClusterID, expectedNames, true) +} + +// DeleteStaleGatewayClients deletes GatewayClients for the remote cluster whose names are not in expectedNames. +func (c *Cluster) DeleteStaleGatewayClients(ctx context.Context, expectedNames []string) error { + return c.deleteStaleGateways(ctx, c.remoteClusterID, expectedNames, false) +} + +func (c *Cluster) deleteStaleGateways(ctx context.Context, remoteClusterID liqov1beta1.ClusterID, expectedNames []string, server bool) error { + expected := make(map[string]struct{}, len(expectedNames)) + for _, n := range expectedNames { + expected[n] = struct{}{} + } + + var toDelete []client.Object + if server { + gwServerList, err := getters.ListGatewayServersByClusterID(ctx, c.local.CRClient, remoteClusterID, c.localNetworkNamespace) + if err != nil { + return err + } + for i := range gwServerList.Items { + if _, ok := expected[gwServerList.Items[i].Name]; !ok { + toDelete = append(toDelete, &gwServerList.Items[i]) + } + } + } else { + gwClientList, err := getters.ListGatewayClientsByClusterID(ctx, c.local.CRClient, remoteClusterID, c.localNetworkNamespace) + if err != nil { + return err + } + for i := range gwClientList.Items { + if _, ok := expected[gwClientList.Items[i].Name]; !ok { + toDelete = append(toDelete, &gwClientList.Items[i]) + } + } + } + + if len(toDelete) == 0 { + return nil + } + + s := c.local.Printer.StartSpinner(fmt.Sprintf("Deleting %d stale gateway(s)", len(toDelete))) + for _, obj := range toDelete { + if err := c.local.CRClient.Delete(ctx, obj); err != nil && !apierrors.IsNotFound(err) { + s.Fail(fmt.Sprintf("An error occurred while deleting stale gateway %q: %v", obj.GetName(), output.PrettyErr(err))) + return err + } + } + s.Success(fmt.Sprintf("Deleted %d stale gateway(s)", len(toDelete))) + return nil +} + +// CheckAlreadyEstablishedForGwServer checks if all expected GatewayServers are already established. +// If existing gateways don't match the expected names, they are stale and will be cleaned up by the caller. +func (c *Cluster) CheckAlreadyEstablishedForGwServer(ctx context.Context, expectedNames []string) (bool, error) { + gwServerList, err := getters.ListGatewayServersByClusterID(ctx, c.local.CRClient, c.remoteClusterID, c.localNetworkNamespace) + if err != nil { return false, err - case apierrors.IsNotFound(err): + } + if len(gwServerList.Items) == 0 { return false, nil - default: - return true, nil } + existing := make(map[string]struct{}, len(gwServerList.Items)) + for i := range gwServerList.Items { + existing[gwServerList.Items[i].Name] = struct{}{} + } + for _, name := range expectedNames { + if _, ok := existing[name]; !ok { + return false, nil + } + } + return true, nil } -// CheckAlreadyEstablishedForGwClient checks if a GatewayClient is already established. -func (c *Cluster) CheckAlreadyEstablishedForGwClient(ctx context.Context) (bool, error) { - _, err := getters.GetGatewayClientByClusterID(ctx, c.local.CRClient, c.remoteClusterID, c.localNetworkNamespace) - switch { - case client.IgnoreNotFound(err) != nil: +// CheckAlreadyEstablishedForGwClient checks if all expected GatewayClients are already established. +// If existing gateways don't match the expected names, they are stale and will be cleaned up by the caller. +func (c *Cluster) CheckAlreadyEstablishedForGwClient(ctx context.Context, expectedNames []string) (bool, error) { + gwClientList, err := getters.ListGatewayClientsByClusterID(ctx, c.local.CRClient, c.remoteClusterID, c.localNetworkNamespace) + if err != nil { return false, err - case apierrors.IsNotFound(err): + } + if len(gwClientList.Items) == 0 { return false, nil - default: - return true, nil } + existing := make(map[string]struct{}, len(gwClientList.Items)) + for i := range gwClientList.Items { + existing[gwClientList.Items[i].Name] = struct{}{} + } + for _, name := range expectedNames { + if _, ok := existing[name]; !ok { + return false, nil + } + } + return true, nil } diff --git a/pkg/liqoctl/network/handler.go b/pkg/liqoctl/network/handler.go index 67fb087143..3a6c4873c1 100644 --- a/pkg/liqoctl/network/handler.go +++ b/pkg/liqoctl/network/handler.go @@ -16,6 +16,7 @@ package network import ( "context" + "fmt" "time" corev1 "k8s.io/api/core/v1" @@ -24,6 +25,7 @@ import ( liqov1beta1 "github.com/liqotech/liqo/apis/core/v1beta1" networkingv1beta1 "github.com/liqotech/liqo/apis/networking/v1beta1" + gwforge "github.com/liqotech/liqo/pkg/gateway/forge" "github.com/liqotech/liqo/pkg/liqo-controller-manager/networking/forge" "github.com/liqotech/liqo/pkg/liqo-controller-manager/networking/getters" "github.com/liqotech/liqo/pkg/liqoctl/factory" @@ -58,6 +60,7 @@ type Options struct { ClientConnectPort int32 MTU int + Replicas int32 DisableSharingKeys bool } @@ -155,58 +158,82 @@ func (o *Options) RunConnect(ctx context.Context) error { return err } - // Check if the reverse Networking is already established on cluster 1 - if established, err := cluster1.CheckAlreadyEstablishedForGwServer(ctx); err != nil { - return err - } else if established { - return nil + // Compute expected gateway names. + serverNames := make([]string, o.Replicas) + clientNames := make([]string, o.Replicas) + for i := range o.Replicas { + serverNames[i] = forgeGatewayName(string(cluster1.localClusterID), i, o.Replicas) + clientNames[i] = forgeGatewayName(string(cluster2.localClusterID), i, o.Replicas) } - // Check if the reverse Networking is already established on cluster 2 - if established, err := cluster2.CheckAlreadyEstablishedForGwClient(ctx); err != nil { + // Clean up stale gateways that don't match the expected names (e.g. scaling from N to M replicas). + if err := cluster2.DeleteStaleGatewayServers(ctx, serverNames); err != nil { return err - } else if established { - return nil } - - // Create gateway server on cluster 2 - gwServer, err := cluster2.EnsureGatewayServer(ctx, o.newGatewayServerForgeOptions(o.RemoteFactory.KubeClient, cluster1.localClusterID)) - if err != nil { + if err := cluster1.DeleteStaleGatewayClients(ctx, clientNames); err != nil { return err } - // Wait for the gateway pod to be ready - if err := cluster2.waiter.ForGatewayPodReady(ctx, gwServer); err != nil { + // Check if the reverse Networking is already established on cluster 1 + if established, err := cluster1.CheckAlreadyEstablishedForGwServer(ctx, serverNames); err != nil { return err + } else if established { + return nil } - // Wait for the endpoint status of the gateway server to be set - if err := cluster2.waiter.ForGatewayServerStatusEndpoint(ctx, gwServer); err != nil { + // Check if the reverse Networking is already established on cluster 2 + if established, err := cluster2.CheckAlreadyEstablishedForGwClient(ctx, clientNames); err != nil { return err + } else if established { + return nil } - // Create gateway client on cluster 1 - - // By default address and port used by the GatewayClient are the ones written in the endpoint field of the status of the GatewayServer, - // unless address or port are manually overwritten - endpoint := gwServer.Status.Endpoint - if o.ClientConnectAddress != "" { - endpoint.Addresses = []string{o.ClientConnectAddress} + // Create N gateway servers on cluster 2, one per replica. + var gwServers []*networkingv1beta1.GatewayServer + for i := range o.Replicas { + name := serverNames[i] + gwServer, err := cluster2.EnsureGatewayServer(ctx, name, o.newGatewayServerForgeOptions(o.RemoteFactory.KubeClient, cluster1.localClusterID)) + if err != nil { + return err + } + gwServers = append(gwServers, gwServer) } - if o.ClientConnectPort != 0 { - endpoint.Port = o.ClientConnectPort + // Wait for all gateway pods to be ready and endpoints to be set. + for _, gwServer := range gwServers { + if err := cluster2.waiter.ForGatewayPodReady(ctx, gwServer); err != nil { + return err + } + if err := cluster2.waiter.ForGatewayServerStatusEndpoint(ctx, gwServer); err != nil { + return err + } } - gwClient, err := cluster1.EnsureGatewayClient(ctx, - o.newGatewayClientForgeOptions(o.LocalFactory.KubeClient, cluster2.localClusterID, endpoint)) - if err != nil { - return err + // Create N gateway clients on cluster 1, one per server. + var gwClients []*networkingv1beta1.GatewayClient + for i, gwServer := range gwServers { + endpoint := gwServer.Status.Endpoint + if o.ClientConnectAddress != "" { + endpoint.Addresses = []string{o.ClientConnectAddress} + } + if o.ClientConnectPort != 0 { + endpoint.Ports = []int32{o.ClientConnectPort} + } + + gwClient, err := cluster1.EnsureGatewayClient(ctx, + clientNames[i], + o.newGatewayClientForgeOptions(o.LocalFactory.KubeClient, cluster2.localClusterID, endpoint)) + if err != nil { + return err + } + gwClients = append(gwClients, gwClient) } - // Wait for the gateway pod to be ready - if err := cluster1.waiter.ForGatewayPodReady(ctx, gwClient); err != nil { - return err + // Wait for all gateway client pods to be ready. + for _, gwClient := range gwClients { + if err := cluster1.waiter.ForGatewayPodReady(ctx, gwClient); err != nil { + return err + } } // If sharing keys is disabled, return immediately @@ -214,53 +241,52 @@ func (o *Options) RunConnect(ctx context.Context) error { return nil } - // Wait for gateway server to set secret reference (containing the server public key) in the status - err = cluster2.waiter.ForGatewayServerSecretRef(ctx, gwServer) - if err != nil { - return err - } - keyServer, err := getters.ExtractKeyFromSecretRef(ctx, cluster2.local.CRClient, gwServer.Status.SecretRef) - if err != nil { - return err - } - - // Create PublicKey of gateway server on cluster 1 - if err := cluster1.EnsurePublicKey(ctx, cluster2.localClusterID, keyServer, gwClient); err != nil { - return err - } + // Exchange keys for each gateway pair. + for i, gwServer := range gwServers { + gwClient := gwClients[i] - // Wait for gateway client to set secret reference (containing the client public key) in the status - err = cluster1.waiter.ForGatewayClientSecretRef(ctx, gwClient) - if err != nil { - return err - } - keyClient, err := getters.ExtractKeyFromSecretRef(ctx, cluster1.local.CRClient, gwClient.Status.SecretRef) - if err != nil { - return err - } + // Wait for gateway server to set secret reference (containing the server public key) in the status + err = cluster2.waiter.ForGatewayServerSecretRef(ctx, gwServer) + if err != nil { + return err + } + keyServer, err := getters.ExtractKeyFromSecretRef(ctx, cluster2.local.CRClient, gwServer.Status.SecretRef) + if err != nil { + return err + } - // Create PublicKey of gateway client on cluster 2 - if err := cluster2.EnsurePublicKey(ctx, cluster1.localClusterID, keyClient, gwServer); err != nil { - return err - } + // Create PublicKey of gateway server on cluster 1 + if err := cluster1.EnsurePublicKey(ctx, cluster2.localClusterID, keyServer, gwClient); err != nil { + return err + } - if o.Wait { - // Wait for Connections on both cluster to be created. - conn2, err := cluster2.waiter.ForConnection(ctx, gwServer.Namespace, cluster1.localClusterID) + // Wait for gateway client to set secret reference (containing the client public key) in the status + err = cluster1.waiter.ForGatewayClientSecretRef(ctx, gwClient) if err != nil { return err } - conn1, err := cluster1.waiter.ForConnection(ctx, gwClient.Namespace, cluster2.localClusterID) + keyClient, err := getters.ExtractKeyFromSecretRef(ctx, cluster1.local.CRClient, gwClient.Status.SecretRef) if err != nil { return err } - // Wait for Connections on both cluster cluster to be established - if err := cluster1.waiter.ForConnectionEstablished(ctx, conn1); err != nil { + // Create PublicKey of gateway client on cluster 2 + if err := cluster2.EnsurePublicKey(ctx, cluster1.localClusterID, keyClient, gwServer); err != nil { return err } - if err := cluster2.waiter.ForConnectionEstablished(ctx, conn2); err != nil { - return err + } + + if o.Wait { + // Wait for all per-gateway Connections on both clusters to be established. + for _, gwServer := range gwServers { + if err := cluster2.waiter.ForConnectionByName(ctx, gwServer.Namespace, gwforge.GatewayResourceName(gwServer.Name)); err != nil { + return err + } + } + for _, gwClient := range gwClients { + if err := cluster1.waiter.ForConnectionByName(ctx, gwClient.Namespace, gwforge.GatewayResourceName(gwClient.Name)); err != nil { + return err + } } } @@ -344,6 +370,16 @@ func (o *Options) initNetworkConfigs(ctx context.Context, cluster1, cluster2 *Cl return nil } +// forgeGatewayName returns the gateway name. For backward compatibility: +// if single replica -> +// if multiple replicas -> - for the others. +func forgeGatewayName(remoteClusterID string, index, replicas int32) string { + if replicas > 1 && index > 0 { + return fmt.Sprintf("%s-%d", remoteClusterID, index) + } + return remoteClusterID +} + func (o *Options) newGatewayServerForgeOptions(kubeClient kubernetes.Interface, remoteClusterID liqov1beta1.ClusterID) *forge.GwServerOptions { return &forge.GwServerOptions{ KubeClient: kubeClient, diff --git a/pkg/liqoctl/peer/handler.go b/pkg/liqoctl/peer/handler.go index db6da27643..2331c289f6 100644 --- a/pkg/liqoctl/peer/handler.go +++ b/pkg/liqoctl/peer/handler.go @@ -59,6 +59,7 @@ type Options struct { ClientConnectAddress string ClientConnectPort int32 MTU int + GwReplicas int32 // Authentication options CreateResourceSlice bool @@ -158,6 +159,7 @@ func ensureNetworking(ctx context.Context, o *Options) error { ClientConnectPort: o.ClientConnectPort, MTU: o.MTU, + Replicas: o.GwReplicas, DisableSharingKeys: false, } diff --git a/pkg/liqoctl/test/network/check/check.go b/pkg/liqoctl/test/network/check/check.go index c7d9b9f7cc..775547ccd4 100644 --- a/pkg/liqoctl/test/network/check/check.go +++ b/pkg/liqoctl/test/network/check/check.go @@ -53,14 +53,16 @@ func RunChecks(ctx context.Context, cl *client.Client, cfg client.Configs, opts successCountTot += successCount errorCountTot += errorCount - logger.Info("Running checks node to pod") - successCount, errorCount, err = RunChecksNodeToPod(ctx, cl, cfg, opts, totreplicas) - PrintCheckResults(successCount, errorCount, logger) - if err != nil { - return fmt.Errorf("failed to run checks node to pod: %w", err) + if opts.NodePod { + logger.Info("Running checks node to pod") + successCount, errorCount, err = RunChecksNodeToPod(ctx, cl, cfg, opts, totreplicas) + PrintCheckResults(successCount, errorCount, logger) + if err != nil { + return fmt.Errorf("failed to run checks node to pod: %w", err) + } + successCountTot += successCount + errorCountTot += errorCount } - successCountTot += successCount - errorCountTot += errorCount if opts.PodToNodePort { logger.Info("Running checks pod to nodeport") diff --git a/pkg/liqoctl/test/network/check/exec.go b/pkg/liqoctl/test/network/check/exec.go index a713ae645d..21ec1a6a44 100644 --- a/pkg/liqoctl/test/network/check/exec.go +++ b/pkg/liqoctl/test/network/check/exec.go @@ -57,7 +57,7 @@ func RunCheckToTargets(ctx context.Context, cl ctrlclient.Client, cfg *rest.Conf }) if !ok || err != nil { logger.Error(fmt.Sprintf("Curl command failed after %d retries", MaxRetries), logger.Args( - "pod", pods.Items[i].Name, "target", targets[j], "error", err, + "pod", pods.Items[i].Name, "node", pods.Items[i].Spec.NodeName, "target", targets[j], "error", err, )) } successCount, errorCount, err = testutils.ManageResults(opts.Topts.FailFast, err, ok, successCount, errorCount) @@ -84,14 +84,14 @@ func ExecCurl(ctx context.Context, pod *corev1.Pod, clset *kubernetes.Clientset, strings.Contains(stdout, "301 Moved Permanently") { if !quiet { logger.Info("Curl command successful", logger.Args( - "pod", pod.Name, "target", endpoint, + "pod", pod.Name, "node", pod.Spec.NodeName, "target", endpoint, )) } return true, nil } logger.Warn("Curl command failed", logger.Args( - "pod", pod.Name, "target", endpoint, "stderr", stderr, "error", err, + "pod", pod.Name, "node", pod.Spec.NodeName, "target", endpoint, "stderr", stderr, "error", err, )) return false, nil diff --git a/pkg/liqoctl/test/network/check/service.go b/pkg/liqoctl/test/network/check/service.go index 2f35cac30e..40a76a5904 100644 --- a/pkg/liqoctl/test/network/check/service.go +++ b/pkg/liqoctl/test/network/check/service.go @@ -259,18 +259,18 @@ func RunsCheckPodToNodePortServiceWithClient(ctx context.Context, cl ctrlclient. } var successCountTot, errorCountTot int32 - for i := 0; i < int(totreplicas*2); i++ { - for i := range nodes.Items { - if nodes.Items[i].GetLabels()[consts.TypeLabel] == consts.TypeNode { - continue - } - if opts.NodePortNodes == flags.NodePortNodesWorkers && setup.IsNodeControlPlane(nodes.Items[i].Spec.Taints) { - continue - } - if opts.NodePortNodes == flags.NodePortNodesControlPlanes && !setup.IsNodeControlPlane(nodes.Items[i].Spec.Taints) { - continue - } - nodeip := GetNodeAddress(&nodes.Items[i]) + for i := range nodes.Items { + if nodes.Items[i].GetLabels()[consts.TypeLabel] == consts.TypeNode { + continue + } + if opts.NodePortNodes == flags.NodePortNodesWorkers && setup.IsNodeControlPlane(nodes.Items[i].Spec.Taints) { + continue + } + if opts.NodePortNodes == flags.NodePortNodesControlPlanes && !setup.IsNodeControlPlane(nodes.Items[i].Spec.Taints) { + continue + } + nodeip := GetNodeAddress(&nodes.Items[i]) + for j := 0; j < int(totreplicas*2); j++ { successCount, errorCount, err = RunCheckToTargets(ctx, cl, cfg[name], opts, name, []string{fmt.Sprintf("http://%s:%d", nodeip, nodeport)}, false, ExecCurl) successCountTot += successCount diff --git a/pkg/liqoctl/test/network/flags/flags.go b/pkg/liqoctl/test/network/flags/flags.go index a983985742..364c9a38cd 100644 --- a/pkg/liqoctl/test/network/flags/flags.go +++ b/pkg/liqoctl/test/network/flags/flags.go @@ -36,6 +36,8 @@ const ( FlagNamesLoadbalancer FlagNames = "lb" // FlagNamesBasic is the flag that runs only pod-to-pod checks. FlagNamesBasic FlagNames = "basic" + // FlagNamesNodePod is the flag that enables curl from node to pod. + FlagNamesNodePod FlagNames = "node-pod" // FlagNamesPodNodeport is the flag that enables curl from pod to nodeport service. FlagNamesPodNodeport FlagNames = "pod-np" // FlagNamesIP is the flag that enables IP remapping for the tests. @@ -52,6 +54,7 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.Var(&o.NodePortNodes, string(FlagNamesNodeportNodes), "Select nodes type for NodePort tests. Possible values: all, workers, control-planes") fs.BoolVar(&o.LoadBalancer, string(FlagNamesLoadbalancer), false, "Enable curl from external to loadbalancer service") fs.BoolVar(&o.Basic, string(FlagNamesBasic), false, "Run only pod-to-pod checks") + fs.BoolVar(&o.NodePod, string(FlagNamesNodePod), false, "Enable curl from node to pod") fs.BoolVar(&o.PodToNodePort, string(FlagNamesPodNodeport), false, "Enable curl from pod to nodeport service") fs.BoolVar(&o.IPRemapping, string(FlagNamesIP), false, "Enable IP remapping for the tests") } diff --git a/pkg/liqoctl/test/network/flags/options.go b/pkg/liqoctl/test/network/flags/options.go index dd6432ce2e..8cc35dce68 100644 --- a/pkg/liqoctl/test/network/flags/options.go +++ b/pkg/liqoctl/test/network/flags/options.go @@ -81,6 +81,8 @@ type Options struct { // Basic Basic bool + // Enable curl from node to pod + NodePod bool // Enable curl from external to nodeport service NodePortExt bool // Select nodes type for NodePort tests. diff --git a/pkg/liqoctl/test/network/setup/deployment.go b/pkg/liqoctl/test/network/setup/deployment.go index 821998fe2c..8335455566 100644 --- a/pkg/liqoctl/test/network/setup/deployment.go +++ b/pkg/liqoctl/test/network/setup/deployment.go @@ -100,6 +100,23 @@ func CreateDeployment(ctx context.Context, cl ctrlclient.Client, replicas int32, }, Spec: corev1.PodSpec{ HostNetwork: hostnetwork, + Affinity: &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{ + { + Weight: 100, + PodAffinityTerm: corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + PodLabelAppCluster: deploymentName + "-" + suffix, + }, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, + }, Containers: []corev1.Container{ { Name: "netshoot", diff --git a/pkg/liqoctl/test/network/setup/kyverno.go b/pkg/liqoctl/test/network/setup/kyverno.go index a4718a79bc..a86dc984b8 100644 --- a/pkg/liqoctl/test/network/setup/kyverno.go +++ b/pkg/liqoctl/test/network/setup/kyverno.go @@ -28,11 +28,26 @@ import ( "github.com/liqotech/liqo/pkg/liqoctl/test/network/flags" ) +const ( + // kyvernoPolicyVersion is the API version used for Kyverno MutatingAdmissionPolicy-based policies. + kyvernoPolicyVersion = "v1" + // kyvernoExpressionKey is the key used for CEL expression fields in Kyverno policies. + kyvernoExpressionKey = "expression" + // podAntiaffinityPolicyName is the base name of the pod anti-affinity Kyverno policy. + podAntiaffinityPolicyName = "pod-antiaffinity" +) + // KyvernoPolicyGroupVersionResource specifies the group version resource used to register the objects. -var KyvernoPolicyGroupVersionResource = schema.GroupVersionResource{Group: "kyverno.io", Version: "v1", Resource: "policies"} +// This API is available starting from Kyverno v1.13 (MutatingAdmissionPolicy-based NamespacedMutatingPolicy). +// Older Kyverno releases do not expose this resource and are not supported by the network tests. +var KyvernoPolicyGroupVersionResource = schema.GroupVersionResource{ + Group: "policies.kyverno.io", + Version: kyvernoPolicyVersion, + Resource: "namespacedmutatingpolicies", +} // KyvernoPolicyKind is the kind of the Kyverno policy. -const KyvernoPolicyKind = "Policy" +const KyvernoPolicyKind = "NamespacedMutatingPolicy" // IsKyvernoAvailable checks if Kyverno is available. func IsKyvernoAvailable(ctx context.Context, cl *dynamic.DynamicClient) bool { @@ -58,33 +73,32 @@ func createPolicyForCluster(ctx context.Context, dynClient dynamic.Interface, cl return nil } -// CreatePolicy creates the Kyverno policies. +// CreatePolicy removes stale Kyverno mutation policies left by previous liqoctl versions. +// Pod anti-affinity and hostNetwork are now baked into the Deployment specs, so no new +// Kyverno policies need to be created. func CreatePolicy(ctx context.Context, cl *client.Client, opts *flags.Options) error { - var kyvernoNotInstalled bool - printer := opts.Topts.LocalFactory.Printer + names := []string{podAntiaffinityPolicyName, podAntiaffinityPolicyName + "-host"} - if IsKyvernoAvailable(ctx, cl.ConsumerDynamic) { - if err := createPolicyForCluster(ctx, cl.ConsumerDynamic, cl.ConsumerName, "consumer"); err != nil { - return err + deleteStale := func(dyn *dynamic.DynamicClient, clusterType string) error { + if !IsKyvernoAvailable(ctx, dyn) { + return nil } - } else { - kyvernoNotInstalled = true - printer.Logger.Warn("Kyverno not available on consumer, skipping policy creation.") - } - - for k := range cl.Providers { - if IsKyvernoAvailable(ctx, cl.ProvidersDynamic[k]) { - if err := createPolicyForCluster(ctx, cl.ProvidersDynamic[k], k, fmt.Sprintf("provider %q", k)); err != nil { - return err + for _, name := range names { + if err := dyn.Resource(KyvernoPolicyGroupVersionResource).Namespace(NamespaceName). + Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("error deleting stale Kyverno policy %q on %s: %w", name, clusterType, err) } - } else { - kyvernoNotInstalled = true - printer.Logger.Warn(fmt.Sprintf("Kyverno not available on provider %q, skipping policy creation.", k)) } + return nil } - if kyvernoNotInstalled { - printer.Logger.Warn("Pods may not be scheduled on every node. Install Kyverno on all clusters for comprehensive tests.") + if err := deleteStale(cl.ConsumerDynamic, "consumer"); err != nil { + return err + } + for k := range cl.Providers { + if err := deleteStale(cl.ProvidersDynamic[k], fmt.Sprintf("provider %q", k)); err != nil { + return err + } } return nil } @@ -97,7 +111,7 @@ func ForgeKyvernoPodAntiaffinityPolicy(suffix string, hostnetwork bool) *unstruc policy.SetAPIVersion(fmt.Sprintf("%s/%s", KyvernoPolicyGroupVersionResource.Group, KyvernoPolicyGroupVersionResource.Version)) deploymentName := DeploymentName - name := "pod-antiaffinity" + name := podAntiaffinityPolicyName if hostnetwork { deploymentName = DeploymentName + "-host" name += "-host" @@ -106,24 +120,30 @@ func ForgeKyvernoPodAntiaffinityPolicy(suffix string, hostnetwork bool) *unstruc policy.SetName(name) policy.SetNamespace(NamespaceName) + labelValue := deploymentName + "-" + suffix + policy.Object["spec"] = map[string]interface{}{ - "rules": []map[string]interface{}{ - { - "name": name, - "match": map[string]interface{}{ - "any": []map[string]interface{}{ - {"resources": map[string]interface{}{ - "kinds": []string{"Pod"}, - "selector": map[string]interface{}{ - "matchLabels": map[string]string{ - PodLabelAppCluster: deploymentName + "-" + suffix, - }, - }, - }}, - }, + "matchConstraints": map[string]interface{}{ + "resourceRules": []map[string]interface{}{ + { + "apiGroups": []string{""}, + "apiVersions": []string{kyvernoPolicyVersion}, + "operations": []string{"CREATE"}, + "resources": []string{"pods"}, }, - "mutate": map[string]interface{}{ - "patchStrategicMerge": forgeRawPatchStrategicMerge(deploymentName+"-"+suffix, hostnetwork), + }, + }, + "matchConditions": []map[string]interface{}{ + { + "name": "match-app-cluster-label", + kyvernoExpressionKey: fmt.Sprintf("object.metadata.?labels['%s'].orValue('') == '%s'", PodLabelAppCluster, labelValue), + }, + }, + "mutations": []map[string]interface{}{ + { + "patchType": "ApplyConfiguration", + "applyConfiguration": map[string]interface{}{ + kyvernoExpressionKey: forgeApplyConfigurationExpression(labelValue, hostnetwork), }, }, }, @@ -131,27 +151,31 @@ func ForgeKyvernoPodAntiaffinityPolicy(suffix string, hostnetwork bool) *unstruc return policy } -func forgeRawPatchStrategicMerge(labelValue string, hostnetwork bool) map[string]interface{} { - return map[string]interface{}{ - "spec": map[string]interface{}{ - "+(affinity)": map[string]interface{}{ - "+(podAntiAffinity)": map[string]interface{}{ - "+(preferredDuringSchedulingIgnoredDuringExecution)": []map[string]interface{}{ - { - "weight": 100, - "podAffinityTerm": map[string]interface{}{ - "labelSelector": map[string]interface{}{ - "matchLabels": map[string]string{ - PodLabelAppCluster: labelValue, - }, - }, - "topologyKey": "kubernetes.io/hostname", +// forgeApplyConfigurationExpression builds the CEL ApplyConfiguration expression that +// enforces pod anti-affinity (and optionally hostNetwork) on the matched pods. +func forgeApplyConfigurationExpression(labelValue string, hostnetwork bool) string { + hostNetwork := "false" + if hostnetwork { + hostNetwork = "true" + } + return fmt.Sprintf(`Object{ + spec: Object.spec{ + hostNetwork: %s, + affinity: Object.spec.affinity{ + podAntiAffinity: Object.spec.affinity.podAntiAffinity{ + preferredDuringSchedulingIgnoredDuringExecution: [ + Object.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution{ + weight: 100, + podAffinityTerm: Object.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm{ + labelSelector: Object.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.labelSelector{ + matchLabels: {"%s": "%s"} }, - }, - }, - }, - }, - "+(hostNetwork)": hostnetwork, - }, + topologyKey: "kubernetes.io/hostname" + } + } + ] + } + } } +}`, hostNetwork, PodLabelAppCluster, labelValue) } diff --git a/pkg/liqoctl/wait/wait.go b/pkg/liqoctl/wait/wait.go index 2cfece046f..680de77712 100644 --- a/pkg/liqoctl/wait/wait.go +++ b/pkg/liqoctl/wait/wait.go @@ -283,6 +283,24 @@ func (w *Waiter) ForGatewayClientSecretRef(ctx context.Context, gwClient *networ return nil } +// ForConnectionByName waits until the Connection resource with the given name has been created and established. +func (w *Waiter) ForConnectionByName(ctx context.Context, namespace, connectionName string) error { + s := w.Printer.StartSpinner(fmt.Sprintf("Waiting for Connection %q to be established", connectionName)) + err := wait.PollUntilContextCancel(ctx, 1*time.Second, true, func(ctx context.Context) (done bool, err error) { + var conn networkingv1beta1.Connection + if err := w.CRClient.Get(ctx, client.ObjectKey{Name: connectionName, Namespace: namespace}, &conn); err != nil { + return false, client.IgnoreNotFound(err) + } + return conn.Status.Value == networkingv1beta1.Connected, nil + }) + if err != nil { + s.Fail(fmt.Sprintf("Failed waiting for Connection %q to be established: %s", connectionName, output.PrettyErr(err))) + return err + } + s.Success(fmt.Sprintf("Connection %q is established", connectionName)) + return nil +} + // ForConnection waits until the Connection resource has been created. func (w *Waiter) ForConnection(ctx context.Context, namespace string, remoteCluster liqov1beta1.ClusterID) (*networkingv1beta1.Connection, error) { diff --git a/pkg/route/route.go b/pkg/route/route.go index 1cd11bad83..5ba2709cce 100644 --- a/pkg/route/route.go +++ b/pkg/route/route.go @@ -119,6 +119,20 @@ func GetRoutesByTableID(tableID uint32) ([]netlink.Route, error) { }, netlink.RT_FILTER_TABLE) } +// FlushRoutesByTableID deletes all routes in the given table ID. +func FlushRoutesByTableID(tableID uint32) error { + routes, err := GetRoutesByTableID(tableID) + if err != nil { + return fmt.Errorf("listing routes for table %d: %w", tableID, err) + } + for i := range routes { + if err := netlink.RouteDel(&routes[i]); err != nil { + return fmt.Errorf("deleting route %v: %w", routes[i], err) + } + } + return nil +} + // IsEqualRoute checks if the two routes are equal. func IsEqualRoute(route1, route2 *netlink.Route) bool { if route1.Dst != nil && route2.Dst != nil && route1.Dst.String() != route2.Dst.String() { diff --git a/pkg/route/routeconfiguration_controller.go b/pkg/route/routeconfiguration_controller.go index 8c93600d6d..287019139c 100644 --- a/pkg/route/routeconfiguration_controller.go +++ b/pkg/route/routeconfiguration_controller.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "strings" "time" "github.com/vishvananda/netlink" @@ -93,6 +94,7 @@ func (r *RouteConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.R if err = r.Get(ctx, req.NamespacedName, routeconfiguration); err != nil { if apierrors.IsNotFound(err) { klog.V(6).Infof("There is no routeconfiguration %s", req.String()) + r.flushStaleRules(req.Name) return ctrl.Result{}, nil } return ctrl.Result{}, fmt.Errorf("getting routeconfiguration: %w", err) @@ -100,9 +102,10 @@ func (r *RouteConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.R klog.V(4).Infof("Reconciling routeconfiguration %s", req.String()) - defer func() { + //nolint:gocritic // keeping commented code for reference + /* defer func() { err = r.UpdateStatus(ctx, r.EventsRecorder, routeconfiguration, r.PodName, err) - }() + }() */ var tableID uint32 tableID, err = GetTableID(routeconfiguration.Spec.Table.Name) @@ -121,39 +124,29 @@ func (r *RouteConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.R return ctrl.Result{}, nil - case deleting && containsFinalizer && r.EnableFinalizer: - existingRules, err := GetRulesByTableID(tableID) - if err != nil { - return ctrl.Result{}, fmt.Errorf("listing existing rules: %w", err) + case deleting: + // Flush all rules and routes in the table, then remove the table entry. + // Each RouteConfiguration owns a unique table ID (derived from Spec.Table.Name), + // so flushing is safe — no other RouteConfiguration uses this table. + if err = FlushRulesByTableID(tableID); err != nil { + return ctrl.Result{}, fmt.Errorf("flushing rules: %w", err) } - existingRoutes, err := GetRoutesByTableID(tableID) - if err != nil { - return ctrl.Result{}, fmt.Errorf("listing existing routes: %w", err) + if err = FlushRoutesByTableID(tableID); err != nil { + return ctrl.Result{}, fmt.Errorf("flushing routes: %w", err) } - - for i := range routeconfiguration.Spec.Table.Rules { - if err = EnsureRuleAbsence(&routeconfiguration.Spec.Table.Rules[i], existingRules); err != nil { - return ctrl.Result{}, fmt.Errorf("ensuring rule absence: %w", err) - } - if err = EnsureRoutesAbsence(routeconfiguration.Spec.Table.Rules[i].Routes, existingRoutes); err != nil { - return ctrl.Result{}, fmt.Errorf("ensuring routes absence: %w", err) - } - } - if err = EnsureTableAbsence(tableID); err != nil { return ctrl.Result{}, fmt.Errorf("ensuring table absence: %w", err) } - if err = r.ensureRouteConfigurationFinalizerAbsence(ctx, routeconfiguration); err != nil { - return ctrl.Result{}, fmt.Errorf("removing finalizer: %w", err) + if containsFinalizer && r.EnableFinalizer { + if err = r.ensureRouteConfigurationFinalizerAbsence(ctx, routeconfiguration); err != nil { + return ctrl.Result{}, fmt.Errorf("removing finalizer: %w", err) + } } klog.V(2).Infof("RouteConfiguration %s deleted", req.String()) return ctrl.Result{}, nil - - case deleting && !containsFinalizer: - return ctrl.Result{}, nil } existingRules, err := GetRulesByTableID(tableID) @@ -236,6 +229,32 @@ func (r *RouteConfigurationReconciler) SetupWithManager(ctx context.Context, mgr Complete(r) } +// flushStaleRules attempts to flush kernel state when a RouteConfiguration is not found. +// This handles the edge case where the object was deleted before the reconciler could run +// The table name is inferred from the object name by stripping known suffixes. +func (r *RouteConfigurationReconciler) flushStaleRules(name string) { + tableName := name + tableName = strings.TrimSuffix(tableName, "-gw-node") + tableName = strings.TrimSuffix(tableName, "-gw-ext") + + tableID, err := GetTableID(tableName) + if err != nil { + return + } + + klog.V(2).Infof("Flushing stale rules for table %d", tableID) + + if err := FlushRulesByTableID(tableID); err != nil { + klog.Warningf("optimistic cleanup: failed to flush rules for table %d (name %q): %v", tableID, tableName, err) + } + if err := FlushRoutesByTableID(tableID); err != nil { + klog.Warningf("optimistic cleanup: failed to flush routes for table %d (name %q): %v", tableID, tableName, err) + } + if err := EnsureTableAbsence(tableID); err != nil { + klog.Warningf("optimistic cleanup: failed to remove table %d (name %q): %v", tableID, tableName, err) + } +} + func getConditionRef(rcfg *networkingv1beta1.RouteConfiguration, podname string) *networkingv1beta1.RouteConfigurationStatusCondition { var conditionRef *networkingv1beta1.RouteConfigurationStatusCondition for i := range rcfg.Status.Conditions { diff --git a/pkg/route/rule.go b/pkg/route/rule.go index 1af7c6e7df..340b8e229b 100644 --- a/pkg/route/rule.go +++ b/pkg/route/rule.go @@ -113,6 +113,20 @@ func GetRulesByTableID(tableID uint32) ([]netlink.Rule, error) { return rules, nil } +// FlushRulesByTableID deletes all policy routing rules associated with the given table ID. +func FlushRulesByTableID(tableID uint32) error { + rules, err := GetRulesByTableID(tableID) + if err != nil { + return fmt.Errorf("listing rules for table %d: %w", tableID, err) + } + for i := range rules { + if err := netlink.RuleDel(&rules[i]); err != nil { + return fmt.Errorf("deleting rule %v: %w", rules[i], err) + } + } + return nil +} + // ExistsRule checks if the given rule is already present in the rules list. func ExistsRule(rule *networkingv1beta1.Rule, rules []netlink.Rule) (*netlink.Rule, bool, error) { for i := range rules { diff --git a/pkg/tenantNamespace/const.go b/pkg/tenantNamespace/const.go index cba97f060d..6d0dd31483 100644 --- a/pkg/tenantNamespace/const.go +++ b/pkg/tenantNamespace/const.go @@ -16,7 +16,7 @@ package tenantnamespace const ( // NamePrefix is the prefix of all tenant namespace names. - NamePrefix = "liqo-tenant" + NamePrefix = "omni" roleBindingRoot = "liqo-binding" ) diff --git a/pkg/tenantNamespace/tenantNamespaceManager_test.go b/pkg/tenantNamespace/tenantNamespaceManager_test.go index b009f44e89..e052cfcdf1 100644 --- a/pkg/tenantNamespace/tenantNamespaceManager_test.go +++ b/pkg/tenantNamespace/tenantNamespaceManager_test.go @@ -37,7 +37,7 @@ var _ = Describe("TenantNamespace", func() { ns, err := namespaceManager.CreateNamespace(ctx, homeCluster) Expect(err).To(BeNil()) Expect(ns).NotTo(BeNil()) - Expect(strings.HasPrefix(ns.Name, "liqo-tenant-")).To(BeTrue()) + Expect(strings.HasPrefix(ns.Name, NamePrefix+"-")).To(BeTrue()) Expect(ns.Labels).NotTo(BeNil()) _, ok := ns.Labels[consts.TenantNamespaceLabel] @@ -65,7 +65,7 @@ var _ = Describe("TenantNamespace", func() { ns, err := namespaceManager.GetNamespace(ctx, homeCluster) Expect(err).To(BeNil()) Expect(ns).NotTo(BeNil()) - Expect(strings.HasPrefix(ns.Name, "liqo-tenant-")).To(BeTrue()) + Expect(strings.HasPrefix(ns.Name, NamePrefix+"-")).To(BeTrue()) Expect(ns.Labels).NotTo(BeNil()) _, ok := ns.Labels[consts.TenantNamespaceLabel] diff --git a/pkg/utils/cluster_info.go b/pkg/utils/cluster_info.go index d0e720c7ce..a14b70fe34 100644 --- a/pkg/utils/cluster_info.go +++ b/pkg/utils/cluster_info.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "os" + "time" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -110,5 +111,12 @@ func GetRestConfig(configPath string) (config *rest.Config, err error) { // Set to in-cluster config. config, err = rest.InClusterConfig() } + + if err != nil { + return nil, err + } + + config.Timeout = 10 * time.Second + return config, err } diff --git a/pkg/utils/getters/k8sGetters.go b/pkg/utils/getters/k8sGetters.go index bf5b451d0a..ef84c1f5f1 100644 --- a/pkg/utils/getters/k8sGetters.go +++ b/pkg/utils/getters/k8sGetters.go @@ -804,31 +804,48 @@ func GetConnectionByClusterIDInNamespace(ctx context.Context, cl client.Client, } } -// GetGatewaysByClusterID returns both the GatewayServer and GatewayClient resource with the given clusterID, if not -// found it returns a nil pointer. Normally, with a common peering, this function should return only one gateway either -// the client or the server. +// GetGatewaysByClusterID returns all GatewayServer and GatewayClient resources with the given clusterID. +// If none are found, both lists will be empty (not nil). Normally, with a common peering, only one type +// (either client or server) will have entries. func GetGatewaysByClusterID(ctx context.Context, cl client.Client, - remoteClusterID liqov1beta1.ClusterID) (*networkingv1beta1.GatewayServer, *networkingv1beta1.GatewayClient, error) { - gwclient, err := GetGatewayClientByClusterID(ctx, cl, remoteClusterID, corev1.NamespaceAll) - if err != nil && !kerrors.IsNotFound(err) { + remoteClusterID liqov1beta1.ClusterID) (*networkingv1beta1.GatewayServerList, *networkingv1beta1.GatewayClientList, error) { + gwServers, err := ListGatewayServersByClusterID(ctx, cl, remoteClusterID, corev1.NamespaceAll) + if err != nil { return nil, nil, err } - gwserver, err := GetGatewayServerByClusterID(ctx, cl, remoteClusterID, corev1.NamespaceAll) - if err != nil && !kerrors.IsNotFound(err) { + gwClients, err := ListGatewayClientsByClusterID(ctx, cl, remoteClusterID, corev1.NamespaceAll) + if err != nil { return nil, nil, err } - return gwserver, gwclient, nil + return gwServers, gwClients, nil } -// GetGatewayServerByClusterID returns the GatewayServer resource with the given clusterID. +// GetGatewayServerByClusterID returns the first GatewayServer resource with the given clusterID. +// When multiple gateway servers exist (multi-gateway setup), the first one is returned. +// Callers needing all gateway servers should use ListGatewayServersByClusterID instead. // If tenantNamespace is empty this function searches in all the namespaces in the cluster. +// +// Deprecated: This function is deprecated and will be removed in a future release. Use ListGatewayServersByClusterID instead. func GetGatewayServerByClusterID(ctx context.Context, cl client.Client, remoteClusterID liqov1beta1.ClusterID, namespace string) (*networkingv1beta1.GatewayServer, error) { + gwServers, err := ListGatewayServersByClusterID(ctx, cl, remoteClusterID, namespace) + if err != nil { + return nil, err + } + if len(gwServers.Items) == 0 { + return nil, kerrors.NewNotFound(networkingv1beta1.GatewayServerGroupResource, networkingv1beta1.GatewayServerResource) + } + return &gwServers.Items[0], nil +} + +// ListGatewayServersByClusterID returns all GatewayServer resources with the given clusterID. +// If tenantNamespace is empty this function searches in all the namespaces in the cluster. +func ListGatewayServersByClusterID(ctx context.Context, cl client.Client, + remoteClusterID liqov1beta1.ClusterID, namespace string) (*networkingv1beta1.GatewayServerList, error) { var gwServers networkingv1beta1.GatewayServerList - if err := cl.List( - ctx, &gwServers, + if err := cl.List(ctx, &gwServers, client.MatchingLabels{ consts.RemoteClusterID: string(remoteClusterID), }, @@ -836,24 +853,33 @@ func GetGatewayServerByClusterID(ctx context.Context, cl client.Client, ); err != nil { return nil, err } - - switch len(gwServers.Items) { - case 0: - return nil, kerrors.NewNotFound(networkingv1beta1.GatewayServerGroupResource, networkingv1beta1.GatewayServerResource) - case 1: - return &gwServers.Items[0], nil - default: - return nil, fmt.Errorf("multiple GatewayServers found for ForeignCluster %s", remoteClusterID) - } + return &gwServers, nil } -// GetGatewayClientByClusterID returns the GatewayClient resource with the given clusterID. +// GetGatewayClientByClusterID returns the first GatewayClient resource with the given clusterID. +// When multiple gateway clients exist (multi-gateway setup), the first one is returned. +// Callers needing all gateway clients should use ListGatewayClientsByClusterID instead. // If tenantNamespace is empty this function searches in all the namespaces in the cluster. +// +// Deprecated: This function is deprecated and will be removed in a future release. Use ListGatewayClientsByClusterID instead. func GetGatewayClientByClusterID(ctx context.Context, cl client.Client, remoteClusterID liqov1beta1.ClusterID, namespace string) (*networkingv1beta1.GatewayClient, error) { + gwClients, err := ListGatewayClientsByClusterID(ctx, cl, remoteClusterID, namespace) + if err != nil { + return nil, err + } + if len(gwClients.Items) == 0 { + return nil, kerrors.NewNotFound(networkingv1beta1.GatewayClientGroupResource, networkingv1beta1.GatewayClientResource) + } + return &gwClients.Items[0], nil +} + +// ListGatewayClientsByClusterID returns all GatewayClient resources with the given clusterID. +// If tenantNamespace is empty this function searches in all the namespaces in the cluster. +func ListGatewayClientsByClusterID(ctx context.Context, cl client.Client, + remoteClusterID liqov1beta1.ClusterID, namespace string) (*networkingv1beta1.GatewayClientList, error) { var gwClients networkingv1beta1.GatewayClientList - if err := cl.List( - ctx, &gwClients, + if err := cl.List(ctx, &gwClients, client.MatchingLabels{ consts.RemoteClusterID: string(remoteClusterID), }, @@ -861,15 +887,7 @@ func GetGatewayClientByClusterID(ctx context.Context, cl client.Client, ); err != nil { return nil, err } - - switch len(gwClients.Items) { - case 0: - return nil, kerrors.NewNotFound(networkingv1beta1.GatewayClientGroupResource, networkingv1beta1.GatewayClientResource) - case 1: - return &gwClients.Items[0], nil - default: - return nil, fmt.Errorf("multiple GatewayClients found for ForeignCluster %s", remoteClusterID) - } + return &gwClients, nil } // GetPublicKeyByClusterID returns the PublicKey resource with the given clusterID. diff --git a/pkg/utils/ipc/client.go b/pkg/utils/ipc/client.go deleted file mode 100644 index a9437dabaf..0000000000 --- a/pkg/utils/ipc/client.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2019-2026 The Liqo Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ipc - -import ( - "context" - "fmt" - "net" - "time" - - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog/v2" -) - -// Connect connects to the Unix domain socket at the given path and sends the given ID. -func Connect(path, id string) (net.Conn, error) { - var c net.Conn - var err error - klog.Infof("Connecting to %s socket", path) - if err := wait.PollUntilContextTimeout(context.Background(), time.Millisecond*100, time.Second*30, false, func(context.Context) (bool, error) { - c, err = net.Dial("unix", path) - if err != nil { - return false, nil - } - return true, nil - }); err != nil { - return nil, err - } - - klog.Infof("Sending message to %s", path) - if _, err := c.Write([]byte(id)); err != nil { - return nil, err - } - return c, nil -} - -// WaitForStart waits for the start signal from the given ID. -func WaitForStart(id string, conn net.Conn) error { - klog.Infof("Waiting for start message") - buf := make([]byte, 4096) - n, err := conn.Read(buf) - if err != nil { - return err - } - msg := string(buf[:n]) - - if msg != ForgeStartMessage(id) { - return fmt.Errorf("unexpected message: %s", msg) - } - klog.Infof("Received start message") - return nil -} diff --git a/pkg/utils/ipc/server.go b/pkg/utils/ipc/server.go deleted file mode 100644 index 3e519806b0..0000000000 --- a/pkg/utils/ipc/server.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2019-2026 The Liqo Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ipc - -import ( - "net" - "os" - - "k8s.io/klog/v2" -) - -// GuestConnections is a map of guest IDs to connections. -type GuestConnections map[string]net.Conn - -// NewGuestConnections creates a new GuestConnections map from string slice. -func NewGuestConnections(guests []string) GuestConnections { - guestsConnections := make(GuestConnections) - for _, guest := range guests { - guestsConnections[guest] = nil - } - return guestsConnections -} - -// checkAllGuestsConnected checks if all guests are connected. -func checkAllGuestsConnected(guestsConnections GuestConnections) bool { - for _, connected := range guestsConnections { - if connected == nil { - return false - } - } - return true -} - -// addGuestConnection adds a new connection to the guestsConnections map. -func addGuestConnection(guestsConnections GuestConnections, guest string, connection net.Conn) { - guestsConnections[guest] = connection -} - -// CreateListenSocket creates a Unix domain socket and listens for incoming connections. -func CreateListenSocket(path string) (net.Listener, error) { - if err := os.RemoveAll(path); err != nil { - return nil, err - } - - // Create a Unix domain socket and listen for incoming connections. - klog.Infof("Listening on %s socket", path) - socket, err := net.Listen("unix", path) - if err != nil { - return nil, err - } - return socket, nil -} - -// CloseListenSocket closes the listen socket. -func CloseListenSocket(socket net.Listener) { - if err := socket.Close(); err != nil { - klog.Errorf("Error closing socket: %v", err) - os.Exit(1) - } -} - -// WaitAllGuestsConnections waits for all guests to connect to the socket. -func WaitAllGuestsConnections(guestsConnections GuestConnections, socket net.Listener) error { - for !checkAllGuestsConnected(guestsConnections) { - // Accept an incoming connection. - conn, err := socket.Accept() - if err != nil { - return err - } - - // Create a buffer for incoming data. - buf := make([]byte, 4096) - - // Read data from the connection. - n, err := conn.Read(buf) - if err != nil { - return err - } - id := string(buf[:n]) - - addGuestConnection(guestsConnections, id, conn) - - klog.Infof("Guest %s connected\n", id) - } - return nil -} - -// CloseAllGuestsConnections closes all connections in the guestsConnections map. -func CloseAllGuestsConnections(guestsConnections GuestConnections) { - for _, conn := range guestsConnections { - if conn != nil { - if err := conn.Close(); err != nil { - klog.Errorf("Error closing connection: %v", err) - os.Exit(1) - } - } - } -} - -// StartAllGuestsConnections sends a start message to all connections in the guestsConnections map. -func StartAllGuestsConnections(guestsConnections GuestConnections) error { - for guest, conn := range guestsConnections { - if _, err := conn.Write([]byte(ForgeStartMessage(guest))); err != nil { - return err - } - } - klog.Infof("Sent start message to all guests") - return nil -} diff --git a/pkg/virtualKubelet/forge/dra.go b/pkg/virtualKubelet/forge/dra.go new file mode 100644 index 0000000000..190ac9c043 --- /dev/null +++ b/pkg/virtualKubelet/forge/dra.go @@ -0,0 +1,116 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package forge + +import ( + corev1 "k8s.io/api/core/v1" + resourcev1 "k8s.io/api/resource/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/utils/ptr" +) + +// LocalResourceSlice builds the local ResourceSlice from the given remote slice, +// setting its OwnerReference to the local virtual node. The caller is responsible for +// having checked that remote.Spec.NodeName is set and matches a local node. +// Important: this function is based on the assumption that the local node name is the same as the remote one. +func LocalResourceSlice( + remote *resourcev1.ResourceSlice, + localNode *corev1.Node, + labelsNotReflected, annotationsNotReflected []string, +) *resourcev1.ResourceSlice { + return &resourcev1.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: remote.Name, + Labels: labels.Merge( + FilterNotReflected(remote.GetLabels(), labelsNotReflected), + ReflectionLabels(), + ), + Annotations: FilterNotReflected(remote.GetAnnotations(), annotationsNotReflected), + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "Node", + Name: localNode.Name, + UID: localNode.UID, + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(false), + }, + }, + }, + Spec: *remote.Spec.DeepCopy(), + } +} + +// RemoteResourceClaim builds a remote ResourceClaim from a local one, dropping +// status and server-managed metadata. +func RemoteResourceClaim(local *resourcev1.ResourceClaim, remoteNamespace string, + labelsNotReflected, annotationsNotReflected []string) *resourcev1.ResourceClaim { + return &resourcev1.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: local.Name, + Namespace: remoteNamespace, + Labels: labels.Merge( + FilterNotReflected(local.GetLabels(), labelsNotReflected), + ReflectionLabels(), + ), + Annotations: FilterNotReflected(local.GetAnnotations(), annotationsNotReflected), + }, + Spec: *local.Spec.DeepCopy(), + } +} + +// RemoteDeviceClass builds a remote DeviceClass from a local one. DeviceClass is +// cluster-scoped; we keep the same name and copy the spec verbatim. +func RemoteDeviceClass(local *resourcev1.DeviceClass, labelsNotReflected, annotationsNotReflected []string) *resourcev1.DeviceClass { + return &resourcev1.DeviceClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: local.Name, + Labels: labels.Merge( + FilterNotReflected(local.GetLabels(), labelsNotReflected), + ReflectionLabels(), + ), + Annotations: FilterNotReflected(local.GetAnnotations(), annotationsNotReflected), + }, + Spec: *local.Spec.DeepCopy(), + } +} + +// ReferencedDeviceClasses extracts the set of DeviceClass names referenced by a +// ResourceClaim, walking both Exactly and FirstAvailable sub-requests. +func ReferencedDeviceClasses(claim *resourcev1.ResourceClaim) []string { + seen := map[string]struct{}{} + out := []string{} + add := func(name string) { + if name == "" { + return + } + if _, ok := seen[name]; ok { + return + } + seen[name] = struct{}{} + out = append(out, name) + } + for i := range claim.Spec.Devices.Requests { + req := &claim.Spec.Devices.Requests[i] + if req.Exactly != nil { + add(req.Exactly.DeviceClassName) + } + for j := range req.FirstAvailable { + add(req.FirstAvailable[j].DeviceClassName) + } + } + return out +} diff --git a/pkg/virtualKubelet/forge/dra_test.go b/pkg/virtualKubelet/forge/dra_test.go new file mode 100644 index 0000000000..e697d5bf7d --- /dev/null +++ b/pkg/virtualKubelet/forge/dra_test.go @@ -0,0 +1,354 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package forge_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + corev1 "k8s.io/api/core/v1" + resourcev1 "k8s.io/api/resource/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + + "github.com/liqotech/liqo/pkg/utils/testutil" + "github.com/liqotech/liqo/pkg/virtualKubelet/forge" +) + +const ( + barVal = "bar" + bazVal = "baz" + fooVal = "foo" + trueVal = "true" + fakeMutatedValue = "mutated" + fakeGPUClassValue = "gpu-class" + fakeGPUValue = "gpu" + fakeFPGAValue = "fpga" +) + +var _ = Describe("DRA forging", func() { + + Describe("the LocalResourceSlice function", func() { + var ( + remote *resourcev1.ResourceSlice + node *corev1.Node + output *resourcev1.ResourceSlice + opts *forge.ForgingOpts + ) + + BeforeEach(func() { + remote = &resourcev1.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "slice", + Labels: map[string]string{ + fooVal: barVal, + testutil.FakeNotReflectedLabelKey: trueVal, + }, + Annotations: map[string]string{ + barVal: bazVal, + testutil.FakeNotReflectedAnnotKey: trueVal, + }, + }, + Spec: resourcev1.ResourceSliceSpec{ + Driver: "test-driver", + NodeName: ptr.To("remote-node"), + Pool: resourcev1.ResourcePool{ + Name: "pool-a", + Generation: 3, + ResourceSliceCount: 1, + }, + Devices: []resourcev1.Device{{Name: "dev-1"}}, + }, + } + node = &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: LiqoNodeName, + UID: types.UID("node-uid-123"), + }, + } + opts = testutil.FakeForgingOpts() + }) + + JustBeforeEach(func() { + output = forge.LocalResourceSlice(remote, node, opts.LabelsNotReflected, opts.AnnotationsNotReflected) + }) + + It("should preserve the slice name", func() { + Expect(output.Name).To(Equal("slice")) + }) + + It("should merge reflection labels with the originals", func() { + Expect(output.Labels).To(HaveKeyWithValue(fooVal, barVal)) + Expect(output.Labels).To(HaveKeyWithValue(forge.LiqoOriginClusterIDKey, string(LocalClusterID))) + Expect(output.Labels).To(HaveKeyWithValue(forge.LiqoDestinationClusterIDKey, string(RemoteClusterID))) + }) + + It("should filter out the not-reflected label/annotation keys", func() { + Expect(output.Labels).ToNot(HaveKey(testutil.FakeNotReflectedLabelKey)) + Expect(output.Annotations).To(HaveKeyWithValue(barVal, bazVal)) + Expect(output.Annotations).ToNot(HaveKey(testutil.FakeNotReflectedAnnotKey)) + }) + + It("should set a single OwnerReference pointing at the local node", func() { + Expect(output.OwnerReferences).To(HaveLen(1)) + ref := output.OwnerReferences[0] + Expect(ref.APIVersion).To(Equal("v1")) + Expect(ref.Kind).To(Equal("Node")) + Expect(ref.Name).To(Equal(node.Name)) + Expect(ref.UID).To(Equal(node.UID)) + Expect(ref.Controller).To(PointTo(BeTrue())) + Expect(ref.BlockOwnerDeletion).To(PointTo(BeFalse())) + }) + + It("should deep-copy the spec (mutating remote does not affect the output)", func() { + Expect(output.Spec.Driver).To(Equal("test-driver")) + remote.Spec.Driver = fakeMutatedValue + remote.Spec.Devices[0].Name = "mutated-dev" + Expect(output.Spec.Driver).To(Equal("test-driver")) + Expect(output.Spec.Devices[0].Name).To(Equal("dev-1")) + }) + + When("the remote has nil labels and annotations", func() { + BeforeEach(func() { + remote.Labels = nil + remote.Annotations = nil + }) + It("should still produce reflection labels without panicking", func() { + Expect(output.Labels).To(HaveKeyWithValue(forge.LiqoOriginClusterIDKey, string(LocalClusterID))) + Expect(output.Labels).To(HaveKeyWithValue(forge.LiqoDestinationClusterIDKey, string(RemoteClusterID))) + }) + }) + + When("the remote labels conflict with reflection-managed keys", func() { + BeforeEach(func() { + remote.Labels[forge.LiqoOriginClusterIDKey] = "some-other-cluster" + }) + It("should let reflection labels win the merge", func() { + Expect(output.Labels).To(HaveKeyWithValue(forge.LiqoOriginClusterIDKey, string(LocalClusterID))) + }) + }) + }) + + Describe("the RemoteResourceClaim function", func() { + const ( + localNamespace = "local-namespace" + remoteNamespace = "remote-namespace" + ) + + var ( + local *resourcev1.ResourceClaim + output *resourcev1.ResourceClaim + opts *forge.ForgingOpts + ) + + BeforeEach(func() { + local = &resourcev1.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "claim", + Namespace: localNamespace, + Labels: map[string]string{ + fooVal: barVal, + testutil.FakeNotReflectedLabelKey: trueVal, + }, + Annotations: map[string]string{ + barVal: bazVal, + testutil.FakeNotReflectedAnnotKey: trueVal, + }, + }, + Spec: resourcev1.ResourceClaimSpec{ + Devices: resourcev1.DeviceClaim{ + Requests: []resourcev1.DeviceRequest{{ + Name: "req-1", + Exactly: &resourcev1.ExactDeviceRequest{ + DeviceClassName: fakeGPUClassValue, + Count: 1, + }, + }}, + }, + }, + Status: resourcev1.ResourceClaimStatus{ + Allocation: &resourcev1.AllocationResult{ + Devices: resourcev1.DeviceAllocationResult{ + Results: []resourcev1.DeviceRequestAllocationResult{{Driver: "x"}}, + }, + }, + }, + } + opts = testutil.FakeForgingOpts() + }) + + JustBeforeEach(func() { + output = forge.RemoteResourceClaim(local, remoteNamespace, opts.LabelsNotReflected, opts.AnnotationsNotReflected) + }) + + It("should preserve the name and remap the namespace", func() { + Expect(output.Name).To(Equal("claim")) + Expect(output.Namespace).To(Equal(remoteNamespace)) + }) + + It("should merge reflection labels and filter not-reflected keys", func() { + Expect(output.Labels).To(HaveKeyWithValue(fooVal, barVal)) + Expect(output.Labels).To(HaveKeyWithValue(forge.LiqoOriginClusterIDKey, string(LocalClusterID))) + Expect(output.Labels).To(HaveKeyWithValue(forge.LiqoDestinationClusterIDKey, string(RemoteClusterID))) + Expect(output.Labels).ToNot(HaveKey(testutil.FakeNotReflectedLabelKey)) + Expect(output.Annotations).To(HaveKeyWithValue(barVal, bazVal)) + Expect(output.Annotations).ToNot(HaveKey(testutil.FakeNotReflectedAnnotKey)) + }) + + It("should deep-copy the spec", func() { + Expect(output.Spec.Devices.Requests[0].Exactly.DeviceClassName).To(Equal(fakeGPUClassValue)) + local.Spec.Devices.Requests[0].Exactly.DeviceClassName = fakeMutatedValue + Expect(output.Spec.Devices.Requests[0].Exactly.DeviceClassName).To(Equal(fakeGPUClassValue)) + }) + + It("should not propagate status (allocation is owned by remote)", func() { + Expect(output.Status).To(Equal(resourcev1.ResourceClaimStatus{})) + }) + + When("the local has nil labels and annotations", func() { + BeforeEach(func() { + local.Labels = nil + local.Annotations = nil + }) + It("should still produce reflection labels without panicking", func() { + Expect(output.Labels).To(HaveKeyWithValue(forge.LiqoOriginClusterIDKey, string(LocalClusterID))) + }) + }) + }) + + Describe("the RemoteDeviceClass function", func() { + var ( + local *resourcev1.DeviceClass + output *resourcev1.DeviceClass + opts *forge.ForgingOpts + ) + + BeforeEach(func() { + local = &resourcev1.DeviceClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: fakeGPUClassValue, + Labels: map[string]string{ + fooVal: barVal, + testutil.FakeNotReflectedLabelKey: trueVal, + }, + Annotations: map[string]string{ + barVal: bazVal, + testutil.FakeNotReflectedAnnotKey: trueVal, + }, + }, + Spec: resourcev1.DeviceClassSpec{ + Selectors: []resourcev1.DeviceSelector{ + {CEL: &resourcev1.CELDeviceSelector{Expression: "device.attributes.foo == \"bar\""}}, + }, + }, + } + opts = testutil.FakeForgingOpts() + }) + + JustBeforeEach(func() { + output = forge.RemoteDeviceClass(local, opts.LabelsNotReflected, opts.AnnotationsNotReflected) + }) + + It("should preserve the cluster-scoped name", func() { + Expect(output.Name).To(Equal(fakeGPUClassValue)) + Expect(output.Namespace).To(BeEmpty()) + }) + + It("should merge reflection labels and filter not-reflected keys", func() { + Expect(output.Labels).To(HaveKeyWithValue(fooVal, barVal)) + Expect(output.Labels).To(HaveKeyWithValue(forge.LiqoOriginClusterIDKey, string(LocalClusterID))) + Expect(output.Labels).ToNot(HaveKey(testutil.FakeNotReflectedLabelKey)) + Expect(output.Annotations).To(HaveKeyWithValue(barVal, bazVal)) + Expect(output.Annotations).ToNot(HaveKey(testutil.FakeNotReflectedAnnotKey)) + }) + + It("should deep-copy the spec", func() { + Expect(output.Spec.Selectors).To(HaveLen(1)) + local.Spec.Selectors[0].CEL.Expression = fakeMutatedValue + Expect(output.Spec.Selectors[0].CEL.Expression).To(Equal("device.attributes.foo == \"bar\"")) + }) + }) + + Describe("the ReferencedDeviceClasses function", func() { + newClaimWithRequests := func(reqs ...resourcev1.DeviceRequest) *resourcev1.ResourceClaim { + return &resourcev1.ResourceClaim{Spec: resourcev1.ResourceClaimSpec{ + Devices: resourcev1.DeviceClaim{Requests: reqs}, + }} + } + + It("should return the single Exactly DeviceClassName", func() { + out := forge.ReferencedDeviceClasses(newClaimWithRequests( + resourcev1.DeviceRequest{Name: "exactDevice", Exactly: &resourcev1.ExactDeviceRequest{DeviceClassName: fakeGPUValue}}, + )) + Expect(out).To(ConsistOf(fakeGPUValue)) + }) + + It("should deduplicate Exactly entries that reference the same class", func() { + out := forge.ReferencedDeviceClasses(newClaimWithRequests( + resourcev1.DeviceRequest{Name: "ded1", Exactly: &resourcev1.ExactDeviceRequest{DeviceClassName: fakeGPUValue}}, + resourcev1.DeviceRequest{Name: "ded2", Exactly: &resourcev1.ExactDeviceRequest{DeviceClassName: fakeGPUValue}}, + resourcev1.DeviceRequest{Name: "ded3", Exactly: &resourcev1.ExactDeviceRequest{DeviceClassName: fakeFPGAValue}}, + )) + Expect(out).To(ConsistOf(fakeGPUValue, fakeFPGAValue)) + }) + + It("should extract names from FirstAvailable subrequests", func() { + out := forge.ReferencedDeviceClasses(newClaimWithRequests( + resourcev1.DeviceRequest{ + Name: "firstAvailable1", + FirstAvailable: []resourcev1.DeviceSubRequest{ + {Name: "firstAvailableSub1", DeviceClassName: fakeGPUValue}, + {Name: "firstAvailableSub2", DeviceClassName: fakeFPGAValue}, + }, + }, + )) + Expect(out).To(ConsistOf(fakeGPUValue, fakeFPGAValue)) + }) + + It("should deduplicate across Exactly and FirstAvailable", func() { + out := forge.ReferencedDeviceClasses(newClaimWithRequests( + resourcev1.DeviceRequest{Name: "exactDevice1", Exactly: &resourcev1.ExactDeviceRequest{DeviceClassName: fakeGPUValue}}, + resourcev1.DeviceRequest{ + Name: "exactDevice2", + FirstAvailable: []resourcev1.DeviceSubRequest{ + {Name: "exactDevice2Sub1", DeviceClassName: fakeGPUValue}, + {Name: "exactDevice2Sub2", DeviceClassName: fakeFPGAValue}, + }, + }, + )) + Expect(out).To(ConsistOf(fakeGPUValue, fakeFPGAValue)) + }) + + It("should return an empty slice for an empty Requests list", func() { + out := forge.ReferencedDeviceClasses(newClaimWithRequests()) + Expect(out).To(BeEmpty()) + }) + + It("should skip empty-string DeviceClassName entries", func() { + out := forge.ReferencedDeviceClasses(newClaimWithRequests( + resourcev1.DeviceRequest{Name: "emptyClassName", Exactly: &resourcev1.ExactDeviceRequest{DeviceClassName: ""}}, + resourcev1.DeviceRequest{ + Name: "emptyClassName2", + FirstAvailable: []resourcev1.DeviceSubRequest{ + {Name: "emptyClassNameSub1", DeviceClassName: ""}, + {Name: "emptyClassNameSub2", DeviceClassName: fakeFPGAValue}, + }, + }, + )) + Expect(out).To(ConsistOf(fakeFPGAValue)) + }) + }) +}) diff --git a/pkg/virtualKubelet/forge/meta.go b/pkg/virtualKubelet/forge/meta.go index 0038bc90e6..7b04226336 100644 --- a/pkg/virtualKubelet/forge/meta.go +++ b/pkg/virtualKubelet/forge/meta.go @@ -30,6 +30,8 @@ const ( LiqoDestinationClusterIDKey = "offloading.liqo.io/destination" // LiqoOriginClusterNodeName is the name of the node on the origin cluster referenced by the virtual-kubelet. LiqoOriginClusterNodeName = "offloading.liqo.io/nodename" + // LiqoReflectedByLabelKey -> The key of the label identifying objects managed by the local-storage PVC reflector. + LiqoReflectedByLabelKey = "offloading.liqo.io/reflected-by" ) // ReflectionLabels returns the labels assigned to the objects reflected from the local to the remote cluster. diff --git a/pkg/virtualKubelet/forge/pods.go b/pkg/virtualKubelet/forge/pods.go index a2a3778bec..a7dfb4912e 100644 --- a/pkg/virtualKubelet/forge/pods.go +++ b/pkg/virtualKubelet/forge/pods.go @@ -99,6 +99,15 @@ func LocalPodOffloadedLabel(local *corev1.Pod) (*corev1apply.PodApplyConfigurati WithLabels(map[string]string{liqoconst.LocalPodLabelKey: liqoconst.LocalPodLabelValue}), true } +// PreserveResourceClaimStatusesMutator returns a RemotePodStatusMutator that restores +// the local pod's ResourceClaimStatuses after the remote status is applied. These are +// managed by the local kubelet and must not be overwritten with the remote's view. +func PreserveResourceClaimStatusesMutator(local []corev1.PodResourceClaimStatus) RemotePodStatusMutator { + return func(remote *corev1.PodStatus) { + remote.ResourceClaimStatuses = local + } +} + // LocalPodStatus forges the status of the local pod, given the remote one. func LocalPodStatus(remote *corev1.PodStatus, translator PodIPTranslator, restarts int32, mutators ...RemotePodStatusMutator) corev1.PodStatus { // Translate the relevant IPs @@ -195,6 +204,7 @@ func RemoteShadowPod(local *corev1.Pod, remote *offloadingv1beta1.ShadowPod, } mutators = append(mutators, RuntimeClassNameMutator(local, forgingOpts)) + mutators = append(mutators, CastResourcesMutator()) return &offloadingv1beta1.ShadowPod{ ObjectMeta: RemoteObjectMeta(localMetaFiltered, &remote.ObjectMeta), @@ -217,6 +227,8 @@ func RemotePodSpec(creation bool, local, remote *corev1.PodSpec, mutators ...Rem remote.Containers = local.Containers remote.InitContainers = local.InitContainers + remote.Resources = local.Resources + remote.ResourceClaims = local.ResourceClaims remote.Tolerations = RemoteTolerations(local.Tolerations) remote.Volumes = local.Volumes @@ -235,16 +247,15 @@ func RemotePodSpec(creation bool, local, remote *corev1.PodSpec, mutators ...Rem remote.ShareProcessNamespace = local.ShareProcessNamespace remote.Subdomain = local.Subdomain remote.TerminationGracePeriodSeconds = local.TerminationGracePeriodSeconds - remote.TopologySpreadConstraints = local.TopologySpreadConstraints // The information about the service account name is not reflected, since the volume is already // present, and the remote creation would fail as the corresponding service account is not present. remote.AutomountServiceAccountToken = ptr.To(false) - // This fields are currently forced to false, to prevent invasive settings on the remote cluster (which might not work). - remote.HostIPC = false + remote.HostIPC = local.HostIPC + remote.HostPID = local.HostPID + // TODO: Add support for host network remote.HostNetwork = false - remote.HostPID = false // Perform the additional mutations to implement advanced functionalities. for _, mutator := range mutators { @@ -273,6 +284,50 @@ func TerminateContainerState(cs *corev1.ContainerStatus, phase corev1.PodPhase, } } +// CheckDRAClaimStatusesReady returns an error if any ResourceClaimTemplateName +// entry in the pod spec has not yet been resolved by the local kubelet into a +// concrete ResourceClaim name in Status.ResourceClaimStatuses. The caller should +// requeue on non-nil to wait for kubelet population. +func CheckDRAClaimStatusesReady(pod *corev1.Pod) error { + resolved := make(map[string]bool, len(pod.Status.ResourceClaimStatuses)) + for i := range pod.Status.ResourceClaimStatuses { + resolved[pod.Status.ResourceClaimStatuses[i].Name] = true + } + for i := range pod.Spec.ResourceClaims { + rc := &pod.Spec.ResourceClaims[i] + if rc.ResourceClaimTemplateName != nil && !resolved[rc.Name] { + return fmt.Errorf("waiting for ResourceClaimStatus for pod %q claim alias %q", pod.Name, rc.Name) + } + } + return nil +} + +// DRAClaimRewriteMutator returns a RemotePodSpecMutator that rewrites any +// ResourceClaimTemplateName entries in pod.Spec.ResourceClaims to the concrete +// ResourceClaimName created in the local cluster, preventing the remote +// cluster from expanding the template and creating a divergent claim. +func DRAClaimRewriteMutator(statuses []corev1.PodResourceClaimStatus) RemotePodSpecMutator { + return func(remote *corev1.PodSpec) { + resolved := make(map[string]string, len(statuses)) + for i := range statuses { + s := &statuses[i] + if s.ResourceClaimName != nil { + resolved[s.Name] = *s.ResourceClaimName + } + } + + for i := range remote.ResourceClaims { + rc := &remote.ResourceClaims[i] + if rc.ResourceClaimTemplateName != nil { + if name, ok := resolved[rc.Name]; ok { + rc.ResourceClaimName = &name + rc.ResourceClaimTemplateName = nil + } + } + } + } +} + // APIServerSupportMutator is a mutator which implements the support to enable offloaded pods to interact back with the local Kubernetes API server. func APIServerSupportMutator(apiServerSupport APIServerSupportType, localAnnotations map[string]string, saName string, saSecretRetriever SASecretRetriever, kubernetesServiceIPRetriever KubernetesServiceIPGetter, @@ -454,6 +509,31 @@ func FilterAntiAffinityLabels(labels map[string]string, whitelist string) map[st appsv1.DefaultDeploymentUniqueLabelKey, appsv1.StatefulSetPodNameLabel)) } +// CastResourcesMutator is a mutator which strips cast.ai-related resource requests/limits +// (e.g. scheduling.cast.ai/network-bandwidth) from containers, as these are already reflected on virtual node. +func CastResourcesMutator() RemotePodSpecMutator { + stripCastResources := func(rl corev1.ResourceList) corev1.ResourceList { + for k := range rl { + if strings.Contains(string(k), "cast.ai/") { + delete(rl, k) + } + } + return rl + } + + stripContainers := func(containers []corev1.Container) { + for i := range containers { + containers[i].Resources.Requests = stripCastResources(containers[i].Resources.Requests) + containers[i].Resources.Limits = stripCastResources(containers[i].Resources.Limits) + } + } + + return func(remote *corev1.PodSpec) { + stripContainers(remote.Containers) + stripContainers(remote.InitContainers) + } +} + // RuntimeClassNameMutator is a mutator which implements the support to propagate the runtimeclass name. func RuntimeClassNameMutator(local *corev1.Pod, forgingOpts *ForgingOpts) RemotePodSpecMutator { return func(remote *corev1.PodSpec) { diff --git a/pkg/virtualKubelet/forge/pods_test.go b/pkg/virtualKubelet/forge/pods_test.go index 148f135592..fb96cf092c 100644 --- a/pkg/virtualKubelet/forge/pods_test.go +++ b/pkg/virtualKubelet/forge/pods_test.go @@ -36,6 +36,11 @@ import ( "github.com/liqotech/liqo/pkg/virtualKubelet/forge" ) +const ( + fakeNICValue = "nic" + fakeGarbageFromRemoteValue = "garbage-from-remote" +) + var _ = Describe("Pod forging", func() { Translator := func(input string) string { return input + "-reflected" } SASecretRetriever := func(input string) string { return input + "-secret" } @@ -591,6 +596,70 @@ var _ = Describe("Pod forging", func() { }) }) + Describe("the CastResourcesMutator function", func() { + var remote *corev1.PodSpec + + BeforeEach(func() { + remote = &corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "c1", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + "scheduling.cast.ai/network-bandwidth": resource.MustParse("10"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("128Mi"), + "scheduling.cast.ai/network-bandwidth": resource.MustParse("20"), + "cast.ai/foo": resource.MustParse("1"), + }, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: "i1", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + "scheduling.cast.ai/network-bandwidth": resource.MustParse("5"), + }, + Limits: corev1.ResourceList{ + "other.cast.ai/something": resource.MustParse("3"), + }, + }, + }, + }, + } + }) + + JustBeforeEach(func() { forge.CastResourcesMutator()(remote) }) + + It("should strip cast.ai resources from container requests", func() { + Expect(remote.Containers[0].Resources.Requests).To(HaveKey(corev1.ResourceCPU)) + Expect(remote.Containers[0].Resources.Requests).NotTo(HaveKey(corev1.ResourceName("scheduling.cast.ai/network-bandwidth"))) + }) + + It("should strip cast.ai resources from container limits", func() { + Expect(remote.Containers[0].Resources.Limits).To(HaveKey(corev1.ResourceMemory)) + Expect(remote.Containers[0].Resources.Limits).NotTo(HaveKey(corev1.ResourceName("scheduling.cast.ai/network-bandwidth"))) + Expect(remote.Containers[0].Resources.Limits).NotTo(HaveKey(corev1.ResourceName("cast.ai/foo"))) + }) + + It("should strip cast.ai resources from init container requests and limits", func() { + Expect(remote.InitContainers[0].Resources.Requests).To(HaveKey(corev1.ResourceCPU)) + Expect(remote.InitContainers[0].Resources.Requests).NotTo(HaveKey(corev1.ResourceName("scheduling.cast.ai/network-bandwidth"))) + Expect(remote.InitContainers[0].Resources.Limits).NotTo(HaveKey(corev1.ResourceName("other.cast.ai/something"))) + }) + + It("should preserve standard resources", func() { + Expect(remote.Containers[0].Resources.Requests[corev1.ResourceCPU]).To(Equal(resource.MustParse("100m"))) + Expect(remote.Containers[0].Resources.Limits[corev1.ResourceMemory]).To(Equal(resource.MustParse("128Mi"))) + Expect(remote.InitContainers[0].Resources.Requests[corev1.ResourceCPU]).To(Equal(resource.MustParse("50m"))) + }) + }) + Describe("the RemoteContainersAPIServerSupport function", func() { var container corev1.Container var output []corev1.Container @@ -993,4 +1062,158 @@ var _ = Describe("Pod forging", func() { }) }) }) + + Describe("the CheckDRAClaimStatusesReady function", func() { + var pod *corev1.Pod + + BeforeEach(func() { + pod = &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p", Namespace: "ns"}} + }) + + It("should return nil for a pod with no ResourceClaims", func() { + Expect(forge.CheckDRAClaimStatusesReady(pod)).To(Succeed()) + }) + + It("should return nil when all references are direct ResourceClaimName", func() { + pod.Spec.ResourceClaims = []corev1.PodResourceClaim{ + {Name: fakeGPUValue, ResourceClaimName: ptr.To("my-claim")}, + {Name: fakeNICValue, ResourceClaimName: ptr.To("my-claim-2")}, + } + Expect(forge.CheckDRAClaimStatusesReady(pod)).To(Succeed()) + }) + + It("should return nil when every template is matched in Status.ResourceClaimStatuses", func() { + pod.Spec.ResourceClaims = []corev1.PodResourceClaim{ + {Name: fakeGPUValue, ResourceClaimTemplateName: ptr.To("gpu-tpl")}, + } + pod.Status.ResourceClaimStatuses = []corev1.PodResourceClaimStatus{ + {Name: fakeGPUValue, ResourceClaimName: ptr.To("p-gpu-abc123")}, + } + Expect(forge.CheckDRAClaimStatusesReady(pod)).To(Succeed()) + }) + + It("should return an error when a template has no matching status", func() { + pod.Spec.ResourceClaims = []corev1.PodResourceClaim{ + {Name: fakeGPUValue, ResourceClaimTemplateName: ptr.To("gpu-tpl")}, + } + err := forge.CheckDRAClaimStatusesReady(pod) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(fakeGPUValue)) + }) + + It("should return an error when only some of multiple templates are matched", func() { + pod.Spec.ResourceClaims = []corev1.PodResourceClaim{ + {Name: fakeGPUValue, ResourceClaimTemplateName: ptr.To("gpu-tpl")}, + {Name: fakeNICValue, ResourceClaimTemplateName: ptr.To("nic-tpl")}, + } + pod.Status.ResourceClaimStatuses = []corev1.PodResourceClaimStatus{ + {Name: fakeGPUValue, ResourceClaimName: ptr.To("p-gpu-abc")}, + } + err := forge.CheckDRAClaimStatusesReady(pod) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(fakeNICValue)) + }) + + It("should not flag direct refs even when other templates are unresolved", func() { + pod.Spec.ResourceClaims = []corev1.PodResourceClaim{ + {Name: fakeGPUValue, ResourceClaimName: ptr.To("my-claim")}, + {Name: fakeNICValue, ResourceClaimTemplateName: ptr.To("nic-tpl")}, + } + err := forge.CheckDRAClaimStatusesReady(pod) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(fakeNICValue)) + Expect(err.Error()).ToNot(ContainSubstring("alias \"" + fakeGPUValue + "\"")) + }) + }) + + Describe("the DRAClaimRewriteMutator function", func() { + It("should leave direct ResourceClaimName references unchanged", func() { + spec := &corev1.PodSpec{ + ResourceClaims: []corev1.PodResourceClaim{ + {Name: fakeGPUValue, ResourceClaimName: ptr.To("my-claim")}, + }, + } + forge.DRAClaimRewriteMutator(nil)(spec) + Expect(spec.ResourceClaims[0].ResourceClaimName).To(PointTo(Equal("my-claim"))) + Expect(spec.ResourceClaims[0].ResourceClaimTemplateName).To(BeNil()) + }) + + It("should rewrite a template ref to the resolved ResourceClaimName", func() { + spec := &corev1.PodSpec{ + ResourceClaims: []corev1.PodResourceClaim{ + {Name: fakeGPUValue, ResourceClaimTemplateName: ptr.To("gpu-tpl")}, + }, + } + statuses := []corev1.PodResourceClaimStatus{ + {Name: fakeGPUValue, ResourceClaimName: ptr.To("p-gpu-abc")}, + } + forge.DRAClaimRewriteMutator(statuses)(spec) + Expect(spec.ResourceClaims).To(HaveLen(1)) + Expect(spec.ResourceClaims[0].ResourceClaimName).To(PointTo(Equal("p-gpu-abc"))) + Expect(spec.ResourceClaims[0].ResourceClaimTemplateName).To(BeNil()) + }) + + It("should rewrite templates while preserving direct refs in a mixed list", func() { + spec := &corev1.PodSpec{ + ResourceClaims: []corev1.PodResourceClaim{ + {Name: fakeGPUValue, ResourceClaimName: ptr.To("my-claim")}, + {Name: fakeNICValue, ResourceClaimTemplateName: ptr.To("nic-tpl")}, + }, + } + statuses := []corev1.PodResourceClaimStatus{ + {Name: fakeNICValue, ResourceClaimName: ptr.To("p-nic-xyz")}, + } + forge.DRAClaimRewriteMutator(statuses)(spec) + Expect(spec.ResourceClaims).To(HaveLen(2)) + Expect(spec.ResourceClaims[0].ResourceClaimName).To(PointTo(Equal("my-claim"))) + Expect(spec.ResourceClaims[0].ResourceClaimTemplateName).To(BeNil()) + Expect(spec.ResourceClaims[1].ResourceClaimName).To(PointTo(Equal("p-nic-xyz"))) + Expect(spec.ResourceClaims[1].ResourceClaimTemplateName).To(BeNil()) + }) + }) + + Describe("the PreserveResourceClaimStatusesMutator function", func() { + It("should overwrite remote ResourceClaimStatuses with the local ones", func() { + localStatuses := []corev1.PodResourceClaimStatus{ + {Name: fakeGPUValue, ResourceClaimName: ptr.To("p-gpu")}, + } + remoteStatus := &corev1.PodStatus{ + ResourceClaimStatuses: []corev1.PodResourceClaimStatus{ + {Name: fakeGarbageFromRemoteValue}, + }, + } + forge.PreserveResourceClaimStatusesMutator(localStatuses)(remoteStatus) + Expect(remoteStatus.ResourceClaimStatuses).To(Equal(localStatuses)) + }) + + It("should clear remote ResourceClaimStatuses", func() { + remoteStatus := &corev1.PodStatus{ + ResourceClaimStatuses: []corev1.PodResourceClaimStatus{ + {Name: fakeGarbageFromRemoteValue}, + }, + } + forge.PreserveResourceClaimStatusesMutator(nil)(remoteStatus) + Expect(remoteStatus.ResourceClaimStatuses).To(BeNil()) + }) + }) + + Describe("the RemotePodSpec function (DRA ResourceClaims propagation)", func() { + It("should copy local.ResourceClaims into the freshly forged remote spec", func() { + local := &corev1.PodSpec{ + ResourceClaims: []corev1.PodResourceClaim{ + {Name: fakeGPUValue, ResourceClaimName: ptr.To("my-claim")}, + }, + } + remote := &corev1.PodSpec{} + out := forge.RemotePodSpec(true /* creation */, local, remote) + Expect(out.ResourceClaims).To(HaveLen(1)) + Expect(out.ResourceClaims[0].Name).To(Equal(fakeGPUValue)) + Expect(out.ResourceClaims[0].ResourceClaimName).To(PointTo(Equal("my-claim"))) + }) + + It("should leave the remote spec ResourceClaims nil when local has none", func() { + out := forge.RemotePodSpec(true, &corev1.PodSpec{}, &corev1.PodSpec{}) + Expect(out.ResourceClaims).To(BeNil()) + }) + }) }) diff --git a/pkg/virtualKubelet/liqoNodeProvider/conditions.go b/pkg/virtualKubelet/liqoNodeProvider/conditions.go index 3251b7315d..4c1cf3bf51 100644 --- a/pkg/virtualKubelet/liqoNodeProvider/conditions.go +++ b/pkg/virtualKubelet/liqoNodeProvider/conditions.go @@ -42,7 +42,7 @@ func UnknownNodeConditions(cfg *InitConfig) []corev1.NodeCondition { // UpdateNodeCondition updates the specified node condition, depending on the outcome of the specified function. func UpdateNodeCondition(node *corev1.Node, conditionType corev1.NodeConditionType, conditionStatus func() (corev1.ConditionStatus, string, string)) { - condition, found := lookupConditionOrCreateUnknown(node, conditionType) + condition, found := lookupConditionOrCreateUnknown(node.Status.Conditions, conditionType) now := metav1.Now() condition.LastHeartbeatTime = now @@ -120,10 +120,10 @@ func unknownCondition(desired corev1.NodeConditionType) *corev1.NodeCondition { } // lookupCondition retrieves a desired condition from a node object, or nil if not found. -func lookupCondition(node *corev1.Node, desired corev1.NodeConditionType) *corev1.NodeCondition { - for i := range node.Status.Conditions { - if node.Status.Conditions[i].Type == desired { - return &node.Status.Conditions[i] +func lookupCondition(conditions []corev1.NodeCondition, desired corev1.NodeConditionType) *corev1.NodeCondition { + for i := range conditions { + if conditions[i].Type == desired { + return &conditions[i] } } @@ -132,8 +132,8 @@ func lookupCondition(node *corev1.Node, desired corev1.NodeConditionType) *corev // lookupConditionOrCreateUnknown retrieves a desired condition from a node object, or create a new one (with unknown status) // if not found. An additional boolean field specifies whether the condition was found or not. -func lookupConditionOrCreateUnknown(node *corev1.Node, desired corev1.NodeConditionType) (*corev1.NodeCondition, bool) { - if condition := lookupCondition(node, desired); condition != nil { +func lookupConditionOrCreateUnknown(conditions []corev1.NodeCondition, desired corev1.NodeConditionType) (*corev1.NodeCondition, bool) { + if condition := lookupCondition(conditions, desired); condition != nil { return condition, true } diff --git a/pkg/virtualKubelet/liqoNodeProvider/conditions_test.go b/pkg/virtualKubelet/liqoNodeProvider/conditions_test.go index c996f06090..b9ba783da3 100644 --- a/pkg/virtualKubelet/liqoNodeProvider/conditions_test.go +++ b/pkg/virtualKubelet/liqoNodeProvider/conditions_test.go @@ -63,7 +63,7 @@ var _ = Describe("Liqo node conditions generation", func() { return status, "reason", "message" } UpdateNodeCondition(&node, corev1.NodeReady, updater) - modified = lookupCondition(&node, corev1.NodeReady) + modified = lookupCondition(node.Status.Conditions, corev1.NodeReady) }) DescribeBodyUpdated := func() func() { @@ -229,13 +229,13 @@ var _ = Describe("Liqo node conditions generation", func() { When("the condition exists", func() { It("should return the expected value", func() { - Expect(*lookupCondition(&node, corev1.NodeReady)).To(Equal(desired)) + Expect(*lookupCondition(node.Status.Conditions, corev1.NodeReady)).To(Equal(desired)) }) }) When("the condition does not exist", func() { It("should return return nil", func() { - Expect(lookupCondition(&node, corev1.NodeNetworkUnavailable)).To(BeNil()) + Expect(lookupCondition(node.Status.Conditions, corev1.NodeNetworkUnavailable)).To(BeNil()) }) }) }) @@ -256,7 +256,7 @@ var _ = Describe("Liqo node conditions generation", func() { }) JustBeforeEach(func() { - retrieved, found = lookupConditionOrCreateUnknown(&node, key) + retrieved, found = lookupConditionOrCreateUnknown(node.Status.Conditions, key) }) When("the condition exists", func() { diff --git a/pkg/virtualKubelet/liqoNodeProvider/nodeProvider.go b/pkg/virtualKubelet/liqoNodeProvider/nodeProvider.go index 28febf46c9..c43f92dd78 100644 --- a/pkg/virtualKubelet/liqoNodeProvider/nodeProvider.go +++ b/pkg/virtualKubelet/liqoNodeProvider/nodeProvider.go @@ -37,6 +37,7 @@ type LiqoNodeProvider struct { localClient kubernetes.Interface remoteDiscoveryClient discovery.DiscoveryInterface dynClient dynamic.Interface + remoteDynClient dynamic.Interface node *corev1.Node terminating bool @@ -54,6 +55,8 @@ type LiqoNodeProvider struct { networkModuleEnabled *bool // nil = ForeignCluster not yet observed networkReady bool + watchRemoteNode bool + remoteNodeStatus corev1.NodeStatus onNodeChangeCallback func(*corev1.Node) updateMutex sync.Mutex @@ -101,8 +104,10 @@ func (p *LiqoNodeProvider) NotifyNodeStatus(_ context.Context, f func(*corev1.No // IsTerminating indicates if the node is in terminating (and in the draining phase). func (p *LiqoNodeProvider) IsTerminating() bool { + klog.Infof("Trying to check if the node is terminating") p.updateMutex.Lock() defer p.updateMutex.Unlock() + klog.Infof("Node is terminating: %v", p.terminating) return p.terminating } @@ -114,7 +119,7 @@ func (p *LiqoNodeProvider) GetNode() *corev1.Node { // hydrate pre-populates the in-memory node state from the available source objects. // It does not publish node updates and does not patch the local Kubernetes Node object. func (p *LiqoNodeProvider) hydrate( - foreignCluster *liqov1beta1.ForeignCluster, virtualNode *offloadingv1beta1.VirtualNode, + foreignCluster *liqov1beta1.ForeignCluster, remoteNode *corev1.Node, virtualNode *offloadingv1beta1.VirtualNode, ) { p.updateMutex.Lock() defer p.updateMutex.Unlock() @@ -126,6 +131,9 @@ func (p *LiqoNodeProvider) hydrate( if foreignCluster != nil { p.applyForeignCluster(foreignCluster) } + if remoteNode != nil { + p.remoteNodeStatus = remoteNode.Status + } p.recomputeNodeState() } diff --git a/pkg/virtualKubelet/liqoNodeProvider/nodeProvider_test.go b/pkg/virtualKubelet/liqoNodeProvider/nodeProvider_test.go index ad7ee458ab..c390dd0170 100644 --- a/pkg/virtualKubelet/liqoNodeProvider/nodeProvider_test.go +++ b/pkg/virtualKubelet/liqoNodeProvider/nodeProvider_test.go @@ -345,7 +345,7 @@ var _ = Describe("NodeProvider", func() { }), ) - It("hydrates the node from virtual node and foreign cluster", func() { + It("hydrates the node from virtual node, foreign cluster and remote node", func() { virtualNode := &offloadingv1beta1.VirtualNode{ ObjectMeta: metav1.ObjectMeta{ Name: nodeName, @@ -391,6 +391,25 @@ var _ = Describe("NodeProvider", func() { }) foreignCluster.Status.ObservedGeneration = 1 + remoteNode := &v1.Node{ + Status: v1.NodeStatus{ + Conditions: []v1.NodeCondition{ + {Type: v1.NodeReady, Status: v1.ConditionTrue}, + {Type: v1.NodeMemoryPressure, Status: v1.ConditionFalse}, + {Type: v1.NodeDiskPressure, Status: v1.ConditionTrue}, + {Type: v1.NodePIDPressure, Status: v1.ConditionFalse}, + }, + NodeInfo: v1.NodeSystemInfo{ + Architecture: "arm64", + OperatingSystem: "linux", + KernelVersion: "6.8.0", + KubeletVersion: "v1.31.0", + ContainerRuntimeVersion: "containerd://2.0.0", + OSImage: "Talos Linux", + }, + }, + } + hydratedProvider := NewLiqoNodeProvider(&InitConfig{ NodeName: nodeName, Namespace: kubeletNamespace, @@ -401,6 +420,7 @@ var _ = Describe("NodeProvider", func() { CheckNetworkStatus: true, VirtualNode: virtualNode, ForeignCluster: foreignCluster, + RemoteNode: remoteNode, }) hydratedNode := hydratedProvider.GetNode() @@ -423,12 +443,13 @@ var _ = Describe("NodeProvider", func() { Expect(hydratedNode.Status.Allocatable.Cpu().Cmp(resource.MustParse("2"))).To(Equal(0)) Expect(hydratedNode.Status.Allocatable.Memory().Cmp(resource.MustParse("3Gi"))).To(Equal(0)) Expect(hydratedNode.Status.Images).To(Equal(virtualNode.Spec.Images)) + Expect(hydratedNode.Status.NodeInfo).To(Equal(remoteNode.Status.NodeInfo)) - Expect(lookupCondition(hydratedNode, v1.NodeReady).Status).To(Equal(v1.ConditionTrue)) - Expect(lookupCondition(hydratedNode, v1.NodeMemoryPressure).Status).To(Equal(v1.ConditionFalse)) - Expect(lookupCondition(hydratedNode, v1.NodeDiskPressure).Status).To(Equal(v1.ConditionFalse)) - Expect(lookupCondition(hydratedNode, v1.NodePIDPressure).Status).To(Equal(v1.ConditionFalse)) - Expect(lookupCondition(hydratedNode, v1.NodeNetworkUnavailable).Status).To(Equal(v1.ConditionFalse)) + Expect(lookupCondition(hydratedNode.Status.Conditions, v1.NodeReady).Status).To(Equal(v1.ConditionTrue)) + Expect(lookupCondition(hydratedNode.Status.Conditions, v1.NodeMemoryPressure).Status).To(Equal(v1.ConditionFalse)) + Expect(lookupCondition(hydratedNode.Status.Conditions, v1.NodeDiskPressure).Status).To(Equal(v1.ConditionTrue)) + Expect(lookupCondition(hydratedNode.Status.Conditions, v1.NodePIDPressure).Status).To(Equal(v1.ConditionFalse)) + Expect(lookupCondition(hydratedNode.Status.Conditions, v1.NodeNetworkUnavailable).Status).To(Equal(v1.ConditionFalse)) }) It("hydrates the node when virtual node labels and annotations are nil", func() { @@ -457,6 +478,7 @@ var _ = Describe("NodeProvider", func() { CheckNetworkStatus: true, VirtualNode: virtualNode, ForeignCluster: nil, + RemoteNode: nil, }) hydratedNode := hydratedProvider.GetNode() diff --git a/pkg/virtualKubelet/liqoNodeProvider/reconciler.go b/pkg/virtualKubelet/liqoNodeProvider/reconciler.go index 5538e94c2f..78bbf6fc79 100644 --- a/pkg/virtualKubelet/liqoNodeProvider/reconciler.go +++ b/pkg/virtualKubelet/liqoNodeProvider/reconciler.go @@ -30,10 +30,12 @@ import ( func (p *LiqoNodeProvider) reconcileNodeFromNode(_ watch.Event) error { // enforce the node to be the same as the one we are managing + klog.V(4).Info("reconciling node from local node event") return p.updateNode() } func (p *LiqoNodeProvider) reconcileNodeFromVirtualNode(event watch.Event) error { + klog.V(4).Info("reconciling node from virtual node event") ctx := context.Background() var virtualNode offloadingv1beta1.VirtualNode unstruct, ok := event.Object.(*unstructured.Unstructured) @@ -49,11 +51,11 @@ func (p *LiqoNodeProvider) reconcileNodeFromVirtualNode(event watch.Event) error klog.Errorf("node update from VirtualNode %v failed for reason %v; retry...", virtualNode.Name, err) return err } - klog.Info("node correctly updated from VirtualNode") return nil } func (p *LiqoNodeProvider) reconcileNodeFromForeignCluster(event watch.Event) error { + klog.V(4).Info("reconciling node from foreigncluster event") var fc liqov1beta1.ForeignCluster unstruct, ok := event.Object.(*unstructured.Unstructured) if !ok { @@ -83,6 +85,25 @@ func (p *LiqoNodeProvider) reconcileNodeFromForeignCluster(event watch.Event) er return nil } +func (p *LiqoNodeProvider) reconcileNodeFromRemoteNode(event watch.Event) error { + klog.V(4).Info("reconciling node from remote node event") + var remoteNode v1.Node + unstruct, ok := event.Object.(*unstructured.Unstructured) + if !ok { + return errors.New("error in casting Remote Node") + } + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstruct.Object, &remoteNode); err != nil { + klog.Error(err) + return err + } + + if err := p.updateFromRemoteNode(&remoteNode); err != nil { + klog.Errorf("node update from remote node %v failed for reason %v; retry...", remoteNode.Name, err) + return err + } + return nil +} + // updateFromVirtualNode gets and updates the node status accordingly. func (p *LiqoNodeProvider) updateFromVirtualNode(ctx context.Context, virtualNode *offloadingv1beta1.VirtualNode) error { @@ -120,6 +141,15 @@ func (p *LiqoNodeProvider) updateFromForeignCluster(foreigncluster *liqov1beta1. return p.updateNode() } +func (p *LiqoNodeProvider) updateFromRemoteNode(remoteNode *v1.Node) error { + p.updateMutex.Lock() + defer p.updateMutex.Unlock() + + p.remoteNodeStatus = remoteNode.Status + + return p.updateNode() +} + func (p *LiqoNodeProvider) updateNode() error { p.recomputeNodeState() diff --git a/pkg/virtualKubelet/liqoNodeProvider/resourceWatcher.go b/pkg/virtualKubelet/liqoNodeProvider/resourceWatcher.go index 8b8f8b81de..a2ebd077f8 100644 --- a/pkg/virtualKubelet/liqoNodeProvider/resourceWatcher.go +++ b/pkg/virtualKubelet/liqoNodeProvider/resourceWatcher.go @@ -65,6 +65,16 @@ func (p *LiqoNodeProvider) StartProvider(ctx context.Context) (ready chan struct runtime.Must(err) } + var remoteNodeInformerFactory dynamicinformer.DynamicSharedInformerFactory + if p.watchRemoteNode { + remoteNodeInformerFactory = dynamicinformer.NewFilteredDynamicSharedInformerFactory( + p.remoteDynClient, p.resyncPeriod, corev1.NamespaceAll, func(opt *metav1.ListOptions) { + opt.FieldSelector = "metadata.name=" + p.nodeName + }) + remoteNodeInformer := remoteNodeInformerFactory.ForResource(corev1.SchemeGroupVersion.WithResource("nodes")).Informer() + _, err = remoteNodeInformer.AddEventHandler(getEventHandler(p.reconcileNodeFromRemoteNode)) + runtime.Must(err) + } ready = make(chan struct{}, 1) go func() { <-ready @@ -72,6 +82,9 @@ func (p *LiqoNodeProvider) StartProvider(ctx context.Context) (ready chan struct if p.checkNetworkStatus { go fcInformerFactory.Start(ctx.Done()) } + if p.watchRemoteNode { + go remoteNodeInformerFactory.Start(ctx.Done()) + } klog.Info("Liqo informers started") }() diff --git a/pkg/virtualKubelet/liqoNodeProvider/setup.go b/pkg/virtualKubelet/liqoNodeProvider/setup.go index 5c2527e95c..0c8c3bcba8 100644 --- a/pkg/virtualKubelet/liqoNodeProvider/setup.go +++ b/pkg/virtualKubelet/liqoNodeProvider/setup.go @@ -65,6 +65,7 @@ type InitConfig struct { VirtualNode *offloadingv1beta1.VirtualNode ForeignCluster *liqov1beta1.ForeignCluster + RemoteNode *corev1.Node } // NewLiqoNodeProvider creates and returns a new LiqoNodeProvider. @@ -73,6 +74,7 @@ func NewLiqoNodeProvider(cfg *InitConfig) *LiqoNodeProvider { localClient: kubernetes.NewForConfigOrDie(cfg.HomeConfig), remoteDiscoveryClient: discovery.NewDiscoveryClientForConfigOrDie(cfg.RemoteConfig), dynClient: dynamic.NewForConfigOrDie(cfg.HomeConfig), + remoteDynClient: dynamic.NewForConfigOrDie(cfg.RemoteConfig), node: node(cfg), terminating: false, @@ -80,6 +82,7 @@ func NewLiqoNodeProvider(cfg *InitConfig) *LiqoNodeProvider { networkModuleEnabled: nil, // set once ForeignCluster is first observed networkReady: false, + watchRemoteNode: cfg.RemoteNode != nil, // watch the remote node only if we were able to retrieve its info resyncPeriod: cfg.InformerResyncPeriod, pingDisabled: cfg.PingDisabled, checkNetworkStatus: cfg.CheckNetworkStatus, @@ -90,18 +93,32 @@ func NewLiqoNodeProvider(cfg *InitConfig) *LiqoNodeProvider { tenantNamespace: cfg.Namespace, } - nodeProvider.hydrate(cfg.ForeignCluster, cfg.VirtualNode) + // Set node status and metadata according to the ForeignCluster, RemoteNode and VirtualNode we retrieved at startup, if available. + nodeProvider.hydrate(cfg.ForeignCluster, cfg.RemoteNode, cfg.VirtualNode) return nodeProvider } func node(cfg *InitConfig) *corev1.Node { + // Default values for the NodeInfo fields if they cannot be retrieved from the remote cluster. + nodeInfo := corev1.NodeSystemInfo{ + KubeletVersion: cfg.Version, + Architecture: architecture, + OperatingSystem: linuxos, + } + + providerID := "" + remoteNode := cfg.RemoteNode + if remoteNode != nil && remoteNode.Spec.ProviderID != "" && strings.HasPrefix(remoteNode.Spec.ProviderID, "castai-omni://") { + providerID = remoteNode.Spec.ProviderID + } + lbls := map[string]string{ corev1.LabelHostname: cfg.NodeName, roleLabelKey: roleLabelValue, - corev1.LabelOSStable: strings.ToLower(linuxos), - corev1.LabelArchStable: architecture, + corev1.LabelOSStable: strings.ToLower(nodeInfo.OperatingSystem), + corev1.LabelArchStable: nodeInfo.Architecture, liqoconst.TypeLabel: liqoconst.TypeNode, // Standard label for virtual-kubelet nodes. This is especially useful for @@ -128,14 +145,11 @@ func node(cfg *InitConfig) *corev1.Node { Annotations: annots, }, Spec: corev1.NodeSpec{ - Taints: taints, + ProviderID: providerID, + Taints: taints, }, Status: corev1.NodeStatus{ - NodeInfo: corev1.NodeSystemInfo{ - KubeletVersion: cfg.Version, - Architecture: architecture, - OperatingSystem: linuxos, - }, + NodeInfo: nodeInfo, Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: cfg.InternalIP}}, DaemonEndpoints: corev1.NodeDaemonEndpoints{KubeletEndpoint: corev1.DaemonEndpoint{Port: int32(cfg.DaemonPort)}}, Capacity: corev1.ResourceList{}, diff --git a/pkg/virtualKubelet/liqoNodeProvider/utils.go b/pkg/virtualKubelet/liqoNodeProvider/utils.go index 352df08c60..9beb5476b5 100644 --- a/pkg/virtualKubelet/liqoNodeProvider/utils.go +++ b/pkg/virtualKubelet/liqoNodeProvider/utils.go @@ -117,11 +117,27 @@ func (p *LiqoNodeProvider) recomputeNodeState() { // check the network status only if we have to set the network condition, otherwise consider it ready networkReady := p.networkReady || !shouldSetNetworkCond - resourcesReady := areResourcesReady(p.node.Status.Allocatable) - UpdateNodeCondition(p.node, v1.NodeReady, nodeReadyStatus(resourcesReady && networkReady)) - UpdateNodeCondition(p.node, v1.NodeMemoryPressure, nodeMemoryPressureStatus(!resourcesReady)) - UpdateNodeCondition(p.node, v1.NodeDiskPressure, nodeDiskPressureStatus(!resourcesReady)) - UpdateNodeCondition(p.node, v1.NodePIDPressure, nodePIDPressureStatus(!resourcesReady)) + if p.watchRemoteNode { + conditionStatus := func(condType v1.NodeConditionType) func() (v1.ConditionStatus, string, string) { + cond, _ := lookupConditionOrCreateUnknown(p.remoteNodeStatus.Conditions, condType) + return func() (status v1.ConditionStatus, reason, message string) { + return cond.Status, cond.Reason, cond.Message + } + } + + remoteReadyStatus, _, _ := conditionStatus(v1.NodeReady)() + UpdateNodeCondition(p.node, v1.NodeReady, nodeReadyStatus(remoteReadyStatus == v1.ConditionTrue && networkReady)) + UpdateNodeCondition(p.node, v1.NodeMemoryPressure, conditionStatus(v1.NodeMemoryPressure)) + UpdateNodeCondition(p.node, v1.NodeDiskPressure, conditionStatus(v1.NodeDiskPressure)) + UpdateNodeCondition(p.node, v1.NodePIDPressure, conditionStatus(v1.NodePIDPressure)) + } else { + // Legacy resource setter. + resourcesReady := areResourcesReady(p.node.Status.Allocatable) + UpdateNodeCondition(p.node, v1.NodeReady, nodeReadyStatus(resourcesReady && networkReady)) + UpdateNodeCondition(p.node, v1.NodeMemoryPressure, nodeMemoryPressureStatus(!resourcesReady)) + UpdateNodeCondition(p.node, v1.NodeDiskPressure, nodeDiskPressureStatus(!resourcesReady)) + UpdateNodeCondition(p.node, v1.NodePIDPressure, nodePIDPressureStatus(!resourcesReady)) + } if shouldSetNetworkCond { UpdateNodeCondition(p.node, v1.NodeNetworkUnavailable, nodeNetworkUnavailableStatus(!networkReady)) @@ -130,6 +146,9 @@ func (p *LiqoNodeProvider) recomputeNodeState() { } p.node.Status.Addresses = []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: p.nodeIP}} + if p.watchRemoteNode { + p.node.Status.NodeInfo = p.remoteNodeStatus.NodeInfo + } } func (p *LiqoNodeProvider) patchLabels(ctx context.Context, labels map[string]string) error { diff --git a/pkg/virtualKubelet/provider/provider.go b/pkg/virtualKubelet/provider/provider.go index 79184f03e9..c3091dc3a1 100644 --- a/pkg/virtualKubelet/provider/provider.go +++ b/pkg/virtualKubelet/provider/provider.go @@ -37,6 +37,7 @@ import ( "github.com/liqotech/liqo/pkg/consts" "github.com/liqotech/liqo/pkg/virtualKubelet/forge" "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/configuration" + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/dra" "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/event" "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/exposition" "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/manager" @@ -159,6 +160,7 @@ func NewLiqoProvider(ctx context.Context, cfg *InitConfig, eb record.EventBroadc ptr.To(cfg.ReflectorsConfigs[resources.ServiceAccount]))). With(storage.NewPersistentVolumeClaimReflector(cfg.VirtualStorageClassName, cfg.RemoteRealStorageClassName, cfg.EnableStorage, ptr.To(cfg.ReflectorsConfigs[resources.PersistentVolumeClaim]))). + With(storage.NewLocalStoragePVCReflector(ptr.To(cfg.ReflectorsConfigs[resources.LocalStoragePVC]))). With(event.NewEventReflector(ptr.To(cfg.ReflectorsConfigs[resources.Event]))). WithNamespaceHandler(namespacemap.NewHandler(localLiqoClient, cfg.Namespace, cfg.InformerResyncPeriod)) @@ -166,6 +168,20 @@ func NewLiqoProvider(ctx context.Context, cfg *InitConfig, eb record.EventBroadc reflectionManager.With(exposition.NewEndpointSliceReflector(cfg.LocalPodCIDRs, ptr.To(cfg.ReflectorsConfigs[resources.EndpointSlice]))) } + draSupported, err := dra.IsDRASupportedOnBothClusters(localClient, remoteClient) + if err != nil { + return nil, errors.Wrap(err, "failed to check whether DRA is supported") + } + if draSupported { + klog.Infof("DRA support detected on both clusters: registering DRA reflectors") + reflectionManager. + With(dra.NewResourceSliceReflector(localClient, remoteClient, cfg.InformerResyncPeriod, + ptr.To(cfg.ReflectorsConfigs[resources.ResourceSlice]))). + With(dra.NewResourceClaimReflector(ptr.To(cfg.ReflectorsConfigs[resources.ResourceClaim]))) + } else { + klog.Info("DRA support is not enabled: resource.k8s.io/v1 is not available on local and/or remote cluster") + } + reflectionManager.Start(ctx) return &LiqoProvider{ diff --git a/pkg/virtualKubelet/reflection/dra/checks.go b/pkg/virtualKubelet/reflection/dra/checks.go new file mode 100644 index 0000000000..d1ffc92939 --- /dev/null +++ b/pkg/virtualKubelet/reflection/dra/checks.go @@ -0,0 +1,63 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dra + +import ( + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes" +) + +// draAPIGroupVersion is the API group/version for Kubernetes Dynamic Resource +// Allocation. Reflectors are only registered when both clusters expose this group. +const draAPIGroupVersion = "resource.k8s.io/v1" + +// IsDRASupportedOnBothClusters returns true only when resource.k8s.io/v1 (with the +// resourceslices, resourceclaims and deviceclasses resources) is available on both the +// local and the remote cluster. +func IsDRASupportedOnBothClusters(local, remote kubernetes.Interface) (bool, error) { + okLocal, err := isDRAAPISupported(local) + if err != nil { + return false, err + } + if !okLocal { + return false, nil + } + return isDRAAPISupported(remote) +} + +func isDRAAPISupported(client kubernetes.Interface) (bool, error) { + res, err := client.Discovery().ServerResourcesForGroupVersion(draAPIGroupVersion) + if err != nil { + if discovery.IsGroupDiscoveryFailedError(err) || kerrors.IsNotFound(err) { + return false, nil + } + return false, err + } + present := map[string]bool{ + "resourceslices": false, + "resourceclaims": false, + "deviceclasses": false, + } + for i := range res.APIResources { + present[res.APIResources[i].Name] = true + } + for _, ok := range present { + if !ok { + return false, nil + } + } + return true, nil +} diff --git a/pkg/virtualKubelet/reflection/dra/checks_test.go b/pkg/virtualKubelet/reflection/dra/checks_test.go new file mode 100644 index 0000000000..b0fc494445 --- /dev/null +++ b/pkg/virtualKubelet/reflection/dra/checks_test.go @@ -0,0 +1,141 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dra_test + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + discoveryfake "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/kubernetes/fake" + clienttesting "k8s.io/client-go/testing" + + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/dra" +) + +const ( + fakeResourceK8sIOV1 = "resource.k8s.io/v1" + fakeDeviceClasses = "deviceclasses" +) + +var _ = Describe("DRA support detection", func() { + + // allDRAResources returns a fully-populated APIResourceList for resource.k8s.io/v1. + allDRAResources := func() *metav1.APIResourceList { + return &metav1.APIResourceList{ + GroupVersion: fakeResourceK8sIOV1, + APIResources: []metav1.APIResource{ + {Name: "resourceslices", Namespaced: false}, + {Name: "resourceclaims", Namespaced: true}, + {Name: fakeDeviceClasses, Namespaced: false}, + }, + } + } + + withResources := func(resources ...*metav1.APIResourceList) *fake.Clientset { + fakeClient := fake.NewClientset() + fakeClient.Discovery().(*discoveryfake.FakeDiscovery).Resources = resources + return fakeClient + } + + Describe("isDRAAPISupported", func() { + It("should return true when all three DRA resources are exposed", func() { + c := withResources(allDRAResources()) + ok, err := dra.IsDRAAPISupported(c) + Expect(err).ToNot(HaveOccurred()) + Expect(ok).To(BeTrue()) + }) + + DescribeTable("should return false when any of the three required resources is missing", + func(missing string) { + rl := allDRAResources() + kept := rl.APIResources[:0] + for _, r := range rl.APIResources { + if r.Name != missing { + kept = append(kept, r) + } + } + rl.APIResources = kept + c := withResources(rl) + + ok, err := dra.IsDRAAPISupported(c) + Expect(err).ToNot(HaveOccurred()) + Expect(ok).To(BeFalse()) + }, + Entry("missing resourceslices", "resourceslices"), + Entry("missing resourceclaims", "resourceclaims"), + Entry("missing deviceclasses", "deviceclasses"), + ) + + It("should return false when the resource.k8s.io/v1 group is not exposed at all", func() { + c := withResources() + ok, err := dra.IsDRAAPISupported(c) + Expect(err).ToNot(HaveOccurred()) + Expect(ok).To(BeFalse()) + }) + + It("should return false when APIResources is empty for the group", func() { + c := withResources(&metav1.APIResourceList{ + GroupVersion: fakeResourceK8sIOV1, + APIResources: []metav1.APIResource{}, + }) + ok, err := dra.IsDRAAPISupported(c) + Expect(err).ToNot(HaveOccurred()) + Expect(ok).To(BeFalse()) + }) + + It("should propagate an unexpected discovery error", func() { + c := fake.NewClientset() + err := errors.New("boom") + c.Discovery().(*discoveryfake.FakeDiscovery).PrependReactor("get", "resource", + func(_ clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, err + }, + ) + ok, err := dra.IsDRAAPISupported(c) + Expect(err).To(MatchError(err)) + Expect(ok).To(BeFalse()) + }) + }) + + Describe("IsDRASupportedOnBothClusters", func() { + It("should return true when both clusters expose DRA", func() { + local := withResources(allDRAResources()) + remote := withResources(allDRAResources()) + ok, err := dra.IsDRASupportedOnBothClusters(local, remote) + Expect(err).ToNot(HaveOccurred()) + Expect(ok).To(BeTrue()) + }) + + It("should return false when local supports DRA but remote does not", func() { + local := withResources(allDRAResources()) + remote := withResources() + ok, err := dra.IsDRASupportedOnBothClusters(local, remote) + Expect(err).ToNot(HaveOccurred()) + Expect(ok).To(BeFalse()) + }) + + It("should return false when remote supports DRA but local does not", func() { + local := withResources() + remote := withResources(allDRAResources()) + ok, err := dra.IsDRASupportedOnBothClusters(local, remote) + Expect(err).ToNot(HaveOccurred()) + Expect(ok).To(BeFalse()) + }) + }) +}) diff --git a/pkg/virtualKubelet/reflection/dra/deviceclass.go b/pkg/virtualKubelet/reflection/dra/deviceclass.go new file mode 100644 index 0000000000..3c381e080a --- /dev/null +++ b/pkg/virtualKubelet/reflection/dra/deviceclass.go @@ -0,0 +1,64 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dra + +import ( + "context" + "fmt" + + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + resourcev1clients "k8s.io/client-go/kubernetes/typed/resource/v1" + "k8s.io/klog/v2" + + "github.com/liqotech/liqo/pkg/virtualKubelet/forge" +) + +// ensureRemoteDeviceClass copies a DeviceClass from local to remote when missing on the +// remote cluster. It returns nil if the class already exists or was successfully created; +// it returns an error if the local class is missing or if the remote create failed for +// reasons other than AlreadyExists. +// +// Per design, reflected DeviceClasses are never garbage-collected: once present on the +// remote, they remain. +func ensureRemoteDeviceClass( + ctx context.Context, + name string, + localClient, remoteClient resourcev1clients.DeviceClassInterface, + labelsNotReflected, annotationsNotReflected []string, +) error { + _, err := remoteClient.Get(ctx, name, metav1.GetOptions{}) + switch { + case err == nil: + return nil + case !kerrors.IsNotFound(err): + return fmt.Errorf("getting remote DeviceClass %q: %w", name, err) + } + + local, err := localClient.Get(ctx, name, metav1.GetOptions{}) + switch { + case kerrors.IsNotFound(err): + return fmt.Errorf("cannot reflect DeviceClass %q that does not exist locally: %w", name, err) + case err != nil: + return fmt.Errorf("getting local DeviceClass %q: %w", name, err) + } + + remote := forge.RemoteDeviceClass(local, labelsNotReflected, annotationsNotReflected) + if _, err := remoteClient.Create(ctx, remote, metav1.CreateOptions{}); err != nil && !kerrors.IsAlreadyExists(err) { + return fmt.Errorf("creating remote DeviceClass %q: %w", name, err) + } + klog.Infof("Reflected DeviceClass %q to remote cluster", name) + return nil +} diff --git a/pkg/virtualKubelet/reflection/dra/deviceclass_test.go b/pkg/virtualKubelet/reflection/dra/deviceclass_test.go new file mode 100644 index 0000000000..eb676db4a0 --- /dev/null +++ b/pkg/virtualKubelet/reflection/dra/deviceclass_test.go @@ -0,0 +1,196 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dra_test + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + resourcev1 "k8s.io/api/resource/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/fake" + clienttesting "k8s.io/client-go/testing" + + "github.com/liqotech/liqo/pkg/utils/testutil" + "github.com/liqotech/liqo/pkg/virtualKubelet/forge" + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/dra" +) + +var _ = Describe("ensureRemoteDeviceClass", func() { + const ( + className = "gpu-class" + deviceClassResource = "deviceclasses" + ) + + var ( + localClient, remoteClient *fake.Clientset + opts *forge.ForgingOpts + ) + + BeforeEach(func() { + localClient = fake.NewClientset() + remoteClient = fake.NewClientset() + opts = testutil.FakeForgingOpts() + }) + + ensureDeviceClass := func() error { + return dra.EnsureRemoteDeviceClass(ctx, className, + localClient.ResourceV1().DeviceClasses(), + remoteClient.ResourceV1().DeviceClasses(), + opts.LabelsNotReflected, opts.AnnotationsNotReflected) + } + + When("the remote DeviceClass already exists", func() { + var createCount int + + BeforeEach(func() { + _, err := remoteClient.ResourceV1().DeviceClasses().Create(ctx, &resourcev1.DeviceClass{ + ObjectMeta: metav1.ObjectMeta{Name: className}, + }, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + createCount = 0 + remoteClient.PrependReactor("create", deviceClassResource, + func(_ clienttesting.Action) (bool, runtime.Object, error) { + createCount++ + return false, nil, nil + }, + ) + }) + + It("should return nil and not issue any Create on the remote", func() { + Expect(ensureDeviceClass()).To(Succeed()) + Expect(createCount).To(Equal(0)) + }) + }) + + When("the remote is missing and the local DeviceClass exists", func() { + BeforeEach(func() { + _, err := localClient.ResourceV1().DeviceClasses().Create(ctx, &resourcev1.DeviceClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: className, + Labels: map[string]string{ + fooVal: barVal, + testutil.FakeNotReflectedLabelKey: trueVal, + }, + Annotations: map[string]string{ + testutil.FakeNotReflectedAnnotKey: trueVal, + }, + }, + }, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should create the remote with reflection labels and filtered keys", func() { + Expect(ensureDeviceClass()).To(Succeed()) + + remote, err := remoteClient.ResourceV1().DeviceClasses().Get(ctx, className, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(remote.Labels).To(HaveKeyWithValue(fooVal, barVal)) + Expect(remote.Labels).To(HaveKeyWithValue(forge.LiqoOriginClusterIDKey, LocalClusterID)) + Expect(remote.Labels).To(HaveKeyWithValue(forge.LiqoDestinationClusterIDKey, RemoteClusterID)) + Expect(remote.Labels).ToNot(HaveKey(testutil.FakeNotReflectedLabelKey)) + Expect(remote.Annotations).ToNot(HaveKey(testutil.FakeNotReflectedAnnotKey)) + }) + }) + + When("both remote and local are missing", func() { + It("should return a NotFound error mentioning that the local class does not exist", func() { + err := ensureDeviceClass() + Expect(err).To(HaveOccurred()) + Expect(kerrors.IsNotFound(err)).To(BeTrue(), + "the error chain must surface NotFound so callers can soft-skip via kerrors.IsNotFound") + Expect(err.Error()).To(ContainSubstring("cannot reflect DeviceClass")) + }) + }) + + When("the remote Get returns a transient error", func() { + BeforeEach(func() { + remoteClient.PrependReactor("get", deviceClassResource, + func(_ clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("remote boom") + }, + ) + }) + It("should propagate the error wrapped with a 'getting remote DeviceClass' prefix", func() { + err := ensureDeviceClass() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("getting remote DeviceClass")) + Expect(err.Error()).To(ContainSubstring("remote boom")) + }) + }) + + When("the local Get returns a transient error", func() { + BeforeEach(func() { + // Remote not present so the function moves on to fetch the local. + localClient.PrependReactor("get", deviceClassResource, + func(_ clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("local boom") + }, + ) + }) + It("should propagate the error wrapped with a 'getting local DeviceClass' prefix", func() { + err := ensureDeviceClass() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("getting local DeviceClass")) + Expect(err.Error()).To(ContainSubstring("local boom")) + }) + }) + + When("the remote Create returns AlreadyExists (concurrent create)", func() { + BeforeEach(func() { + _, err := localClient.ResourceV1().DeviceClasses().Create(ctx, &resourcev1.DeviceClass{ + ObjectMeta: metav1.ObjectMeta{Name: className}, + }, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + remoteClient.PrependReactor("create", deviceClassResource, + func(_ clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, kerrors.NewAlreadyExists(schema.GroupResource{ + Group: "resource.k8s.io", Resource: deviceClassResource, + }, className) + }, + ) + }) + It("should swallow the error and report success", func() { + Expect(ensureDeviceClass()).To(Succeed()) + }) + }) + + When("the remote Create returns an unexpected error", func() { + BeforeEach(func() { + _, err := localClient.ResourceV1().DeviceClasses().Create(ctx, &resourcev1.DeviceClass{ + ObjectMeta: metav1.ObjectMeta{Name: className}, + }, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + remoteClient.PrependReactor("create", deviceClassResource, + func(_ clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("boom while creating") + }, + ) + }) + It("should propagate the error wrapped with a 'creating remote DeviceClass' prefix", func() { + err := ensureDeviceClass() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("creating remote DeviceClass")) + Expect(err.Error()).To(ContainSubstring("boom while creating")) + }) + }) +}) diff --git a/pkg/utils/ipc/doc.go b/pkg/virtualKubelet/reflection/dra/doc.go similarity index 80% rename from pkg/utils/ipc/doc.go rename to pkg/virtualKubelet/reflection/dra/doc.go index 0cbcc36d00..ca414ca182 100644 --- a/pkg/utils/ipc/doc.go +++ b/pkg/virtualKubelet/reflection/dra/doc.go @@ -12,5 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package ipc contains the logic to manage the Inter-Process Communication (IPC) between the gateway replicas. -package ipc +// Package dra provides Dynamic Resource Allocation (DRA) support detection and reflection utilities +// for the Liqo virtual kubelet. +package dra diff --git a/pkg/virtualKubelet/reflection/dra/dra_suite_test.go b/pkg/virtualKubelet/reflection/dra/dra_suite_test.go new file mode 100644 index 0000000000..0257b196d6 --- /dev/null +++ b/pkg/virtualKubelet/reflection/dra/dra_suite_test.go @@ -0,0 +1,101 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dra_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "github.com/liqotech/liqo/pkg/consts" + "github.com/liqotech/liqo/pkg/utils/testutil" + "github.com/liqotech/liqo/pkg/virtualKubelet/forge" + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/options" +) + +const ( + LocalNamespace = "local-namespace" + RemoteNamespace = "remote-namespace" + + LocalClusterID = "local-cluster-id" + RemoteClusterID = "remote-cluster-id" + + LiqoNodeName = "local-node" + LiqoNodeIP = "1.1.1.1" + LiqoNodeUID = types.UID("liqo-node-uid") + + // Shared test constants used across DRA test files. + barVal = "bar" + fooVal = "foo" + trueVal = "true" + fakeClaimName = "req-1" + fakeDriverName = "test.driver" + fakePoolName = "pool" +) + +var ( + // Per-test fake clients. Recreated in BeforeEach so tests are isolated. + localClient, remoteClient *fake.Clientset + + ctx context.Context + cancel context.CancelFunc +) + +func TestDRA(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "DRA Reflection Suite") +} + +var _ = BeforeSuite(func() { + testutil.LogsToGinkgoWriter() + forge.Init(LocalClusterID, RemoteClusterID, LiqoNodeName, LiqoNodeIP) +}) + +var _ = BeforeEach(func() { + ctx, cancel = context.WithCancel(context.Background()) + + // Local cluster preloaded with the virtual node so OwnerReference resolution + // works without each test having to recreate it. + localClient = fake.NewClientset(&corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: LiqoNodeName, + UID: LiqoNodeUID, + Labels: map[string]string{ + corev1.LabelHostname: LiqoNodeName, + consts.TypeLabel: consts.TypeNode, + }, + }, + }) + remoteClient = fake.NewClientset() +}) + +var _ = AfterEach(func() { cancel() }) + +// FakeEventHandler is a no-op handler factory used to bypass real informer +// event-handler wiring in tests. +var FakeEventHandler = func(options.Keyer, ...options.EventFilter) cache.ResourceEventHandler { + return cache.ResourceEventHandlerFuncs{ + AddFunc: func(_ interface{}) {}, + UpdateFunc: func(_, _ interface{}) {}, + DeleteFunc: func(_ interface{}) {}, + } +} diff --git a/pkg/virtualKubelet/reflection/dra/export_test.go b/pkg/virtualKubelet/reflection/dra/export_test.go new file mode 100644 index 0000000000..4a3f45bbc6 --- /dev/null +++ b/pkg/virtualKubelet/reflection/dra/export_test.go @@ -0,0 +1,74 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dra + +import ( + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + corev1listers "k8s.io/client-go/listers/core/v1" + resourcev1listers "k8s.io/client-go/listers/resource/v1" + "k8s.io/client-go/util/workqueue" + + "github.com/liqotech/liqo/pkg/virtualKubelet/forge" +) + +// Test-only exports for private members of the dra package. + +// Handle exposes ResourceSliceReflector.handle for tests. +var Handle = (*ResourceSliceReflector).handle + +// EnqueueRemoteSlicesForNode exposes ResourceSliceReflector.enqueueRemoteSlicesForNode for tests. +var EnqueueRemoteSlicesForNode = (*ResourceSliceReflector).enqueueRemoteSlicesForNode + +// EnsureRemoteDeviceClass exposes the package-private function for tests. +var EnsureRemoteDeviceClass = ensureRemoteDeviceClass + +// IsDRAAPISupported exposes the package-private discovery probe for tests. +var IsDRAAPISupported = isDRAAPISupported + +// NewResourceSliceReflectorForTest builds a ResourceSliceReflector with prebuilt +// listers and clients, bypassing the informer factory wiring done by Start. +func NewResourceSliceReflectorForTest( + localClient, remoteClient kubernetes.Interface, + localSlices, remoteSlices resourcev1listers.ResourceSliceLister, + localNodes corev1listers.NodeLister, + forgingOpts *forge.ForgingOpts, +) *ResourceSliceReflector { + return &ResourceSliceReflector{ + name: ResourceSliceReflectorName, + workers: 1, + localClient: localClient, + remoteClient: remoteClient, + localSlices: localSlices, + remoteSlices: remoteSlices, + localNodes: localNodes, + forgingOpts: forgingOpts, + workqueue: workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[types.NamespacedName](), + workqueue.TypedRateLimitingQueueConfig[types.NamespacedName]{Name: ResourceSliceReflectorName}), + } +} + +// SetForgingOpts mutates the forging options on a reflector after construction. +// Used by tests to swap the opts mid-suite. +func (rsr *ResourceSliceReflector) SetForgingOpts(forgingOpts *forge.ForgingOpts) { + rsr.forgingOpts = forgingOpts +} + +// WorkqueueLen returns the number of items currently queued. Test-only helper +// so suites can assert on enqueue/resync behavior. +func (rsr *ResourceSliceReflector) WorkqueueLen() int { + return rsr.workqueue.Len() +} diff --git a/pkg/virtualKubelet/reflection/dra/resourceclaim.go b/pkg/virtualKubelet/reflection/dra/resourceclaim.go new file mode 100644 index 0000000000..6508fb5ab0 --- /dev/null +++ b/pkg/virtualKubelet/reflection/dra/resourceclaim.go @@ -0,0 +1,201 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dra + +import ( + "context" + "fmt" + + resourcev1 "k8s.io/api/resource/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + resourcev1clients "k8s.io/client-go/kubernetes/typed/resource/v1" + resourcev1listers "k8s.io/client-go/listers/resource/v1" + "k8s.io/klog/v2" + + offloadingv1beta1 "github.com/liqotech/liqo/apis/offloading/v1beta1" + "github.com/liqotech/liqo/pkg/utils/virtualkubelet" + "github.com/liqotech/liqo/pkg/virtualKubelet/forge" + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/generic" + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/manager" + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/options" +) + +const ( + // ResourceClaimReflectorName is the name associated with the ResourceClaim reflector. + ResourceClaimReflectorName = "ResourceClaim" +) + +// ResourceClaimReflector mirrors ResourceClaim objects from the local cluster to the +// remote cluster. The lifecycle of a remote claim is anchored to the local claim: a +// remote claim is deleted only when its local counterpart is deleted. +type ResourceClaimReflector struct { + manager.Reflector +} + +// NamespacedResourceClaimReflector handles ResourceClaim reflection for a single +// (local, remote) namespace pair. +type NamespacedResourceClaimReflector struct { + generic.NamespacedReflector + + localClaims resourcev1listers.ResourceClaimNamespaceLister + remoteClaims resourcev1listers.ResourceClaimNamespaceLister + + remoteClaimsClient resourcev1clients.ResourceClaimInterface + localDeviceClassesClient resourcev1clients.DeviceClassInterface + remoteDeviceClassesClient resourcev1clients.DeviceClassInterface +} + +var _ manager.NamespacedReflector = (*NamespacedResourceClaimReflector)(nil) + +// NewResourceClaimReflector returns a new ResourceClaimReflector. +func NewResourceClaimReflector(reflectorConfig *offloadingv1beta1.ReflectorConfig) manager.Reflector { + r := &ResourceClaimReflector{} + r.Reflector = generic.NewReflector(ResourceClaimReflectorName, + r.NewNamespaced, generic.WithoutFallback(), + reflectorConfig.NumWorkers, offloadingv1beta1.CustomLiqo, generic.ConcurrencyModeAll) + return r +} + +// NewNamespaced returns a new NamespacedResourceClaimReflector instance. +func (rcr *ResourceClaimReflector) NewNamespaced(opts *options.NamespacedOpts) manager.NamespacedReflector { + localClaims := opts.LocalFactory.Resource().V1().ResourceClaims() + remoteClaims := opts.RemoteFactory.Resource().V1().ResourceClaims() + + // Local claim events enqueue the claim name directly. + _, err := localClaims.Informer().AddEventHandler(opts.HandlerFactory(generic.NamespacedKeyer(opts.LocalNamespace))) + utilruntime.Must(err) + // Remote claim events enqueue by name (preserved during reflection). + _, err = remoteClaims.Informer().AddEventHandler(opts.HandlerFactory( + func(obj metav1.Object) []types.NamespacedName { + if !forge.IsReflected(obj) { + return nil + } + return []types.NamespacedName{{Namespace: opts.LocalNamespace, Name: obj.GetName()}} + }, + )) + utilruntime.Must(err) + + return &NamespacedResourceClaimReflector{ + NamespacedReflector: generic.NewNamespacedReflector(opts, ResourceClaimReflectorName), + + localClaims: localClaims.Lister().ResourceClaims(opts.LocalNamespace), + remoteClaims: remoteClaims.Lister().ResourceClaims(opts.RemoteNamespace), + + remoteClaimsClient: opts.RemoteClient.ResourceV1().ResourceClaims(opts.RemoteNamespace), + localDeviceClassesClient: opts.LocalClient.ResourceV1().DeviceClasses(), + remoteDeviceClassesClient: opts.RemoteClient.ResourceV1().DeviceClasses(), + } +} + +// Handle reconciles a single ResourceClaim identified by claimName. +// The remote claim lifecycle is anchored to the local claim: the remote is +// deleted only when the local claim no longer exists. +func (n *NamespacedResourceClaimReflector) Handle(ctx context.Context, claimName string) error { + klog.V(4).Infof("Handling ResourceClaim reflection for local claim %q", n.LocalRef(claimName)) + + local, lerr := n.localClaims.Get(claimName) + if lerr != nil && !kerrors.IsNotFound(lerr) { + return lerr + } + + remote, rerr := n.remoteClaims.Get(claimName) + if rerr != nil && !kerrors.IsNotFound(rerr) { + return rerr + } + + if remote == nil && local == nil { + // No local and no remote: nothing to do. + return nil + } + + // Refuse to mutate a remote claim we don't own. + if rerr == nil && !forge.IsReflected(remote) { + klog.Infof("Skipping reflection of ResourceClaim %q: remote exists and is not managed by Liqo", n.RemoteRef(claimName)) + return nil + } + + // Local claim gone: clean up the remote if we own it. + if kerrors.IsNotFound(lerr) && remote != nil { + klog.Infof("Deleting remote ResourceClaim %q: local no longer exists", n.RemoteRef(claimName)) + return n.DeleteRemote(ctx, n.remoteClaimsClient, ResourceClaimReflectorName, claimName, remote.GetUID()) + } + + // Ensure every DeviceClass referenced by the claim is present on the remote. + for _, dc := range forge.ReferencedDeviceClasses(local) { + err := ensureRemoteDeviceClass(ctx, dc, + n.localDeviceClassesClient, n.remoteDeviceClassesClient, + n.ForgingOpts.LabelsNotReflected, n.ForgingOpts.AnnotationsNotReflected) + switch { + case kerrors.IsNotFound(err): + // Do not reflect the claim if any referenced class is missing locally. + klog.Warningf("Skipping reflection of ResourceClaim %q as it references missing local DeviceClass %q", n.LocalRef(claimName), dc) + return nil + case err != nil: + return fmt.Errorf("ensuring ResourceClaim %q DeviceClass: %w", claimName, err) + } + } + + desired := forge.RemoteResourceClaim(local, n.RemoteNamespace(), + n.ForgingOpts.LabelsNotReflected, n.ForgingOpts.AnnotationsNotReflected) + + if kerrors.IsNotFound(rerr) { + if _, err := n.remoteClaimsClient.Create(ctx, desired, metav1.CreateOptions{}); err != nil && !kerrors.IsAlreadyExists(err) { + klog.Errorf("Failed to create remote ResourceClaim %q: %v", n.RemoteRef(claimName), err) + return fmt.Errorf("creating ResourceClaim %q: %w", n.RemoteRef(claimName), err) + } + klog.Infof("Remote ResourceClaim %q successfully created", n.RemoteRef(claimName)) + return nil + } + + // ResourceClaim.Spec is immutable; only labels/annotations can be updated. + if apiequality.Semantic.DeepEqual(remote.Labels, desired.Labels) && + apiequality.Semantic.DeepEqual(remote.Annotations, desired.Annotations) { + klog.V(4).Infof("Remote ResourceClaim %q is already up-to-date", n.RemoteRef(claimName)) + return nil + } + updated := remote.DeepCopy() + updated.Labels = desired.Labels + updated.Annotations = desired.Annotations + if _, err := n.remoteClaimsClient.Update(ctx, updated, metav1.UpdateOptions{}); err != nil { + klog.Errorf("Failed to update remote ResourceClaim %q metadata: %v", n.RemoteRef(claimName), err) + return fmt.Errorf("updating ResourceClaim %q: %w", n.RemoteRef(claimName), err) + } + return nil +} + +// List returns the union of local and remote reflected claim names so that resync +// catches orphaned remote claims whose local was deleted while the VK was down. +func (n *NamespacedResourceClaimReflector) List() ([]any, error) { + local, lerr := virtualkubelet.List[virtualkubelet.Lister[*resourcev1.ResourceClaim]]( + n.localClaims, + ) + + if lerr != nil { + return nil, fmt.Errorf("listing local ResourceClaims: %w", lerr) + } + + remote, rerr := virtualkubelet.List[virtualkubelet.Lister[*resourcev1.ResourceClaim]]( + n.remoteClaims, + ) + if rerr != nil { + return nil, fmt.Errorf("listing remote ResourceClaims: %w", rerr) + } + + return append(local, remote...), nil +} diff --git a/pkg/virtualKubelet/reflection/dra/resourceclaim_test.go b/pkg/virtualKubelet/reflection/dra/resourceclaim_test.go new file mode 100644 index 0000000000..1f6d9c3ac6 --- /dev/null +++ b/pkg/virtualKubelet/reflection/dra/resourceclaim_test.go @@ -0,0 +1,350 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dra_test + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + resourcev1 "k8s.io/api/resource/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/tools/record" + "k8s.io/utils/trace" + + offloadingv1beta1 "github.com/liqotech/liqo/apis/offloading/v1beta1" + "github.com/liqotech/liqo/pkg/utils/testutil" + "github.com/liqotech/liqo/pkg/virtualKubelet/forge" + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/dra" + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/manager" + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/options" +) + +var _ = Describe("NamespacedResourceClaimReflector", func() { + const ( + claimName = "claim-1" + deviceClassName = "gpu-class" + ) + + var ( + reflector manager.NamespacedReflector + err error + ) + + createLocalClaim := func(c *resourcev1.ResourceClaim) *resourcev1.ResourceClaim { + c.Namespace = LocalNamespace + out, e := localClient.ResourceV1().ResourceClaims(LocalNamespace).Create(ctx, c, metav1.CreateOptions{}) + Expect(e).ToNot(HaveOccurred()) + return out + } + createRemoteClaim := func(c *resourcev1.ResourceClaim) *resourcev1.ResourceClaim { + c.Namespace = RemoteNamespace + out, e := remoteClient.ResourceV1().ResourceClaims(RemoteNamespace).Create(ctx, c, metav1.CreateOptions{}) + Expect(e).ToNot(HaveOccurred()) + return out + } + createLocalDeviceClass := func(name string) { + _, e := localClient.ResourceV1().DeviceClasses().Create(ctx, &resourcev1.DeviceClass{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + }, metav1.CreateOptions{}) + Expect(e).ToNot(HaveOccurred()) + } + createRemoteDeviceClass := func(name string) *resourcev1.DeviceClass { + out, e := remoteClient.ResourceV1().DeviceClasses().Create(ctx, &resourcev1.DeviceClass{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + }, metav1.CreateOptions{}) + Expect(e).ToNot(HaveOccurred()) + return out + } + + mkBasicLocalClaim := func() *resourcev1.ResourceClaim { + return &resourcev1.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Labels: map[string]string{ + fooVal: barVal, + testutil.FakeNotReflectedLabelKey: trueVal, + }, + }, + Spec: resourcev1.ResourceClaimSpec{ + Devices: resourcev1.DeviceClaim{ + Requests: []resourcev1.DeviceRequest{{ + Name: fakeClaimName, + Exactly: &resourcev1.ExactDeviceRequest{DeviceClassName: deviceClassName, Count: 1}, + }}, + }, + }, + } + } + + JustBeforeEach(func() { + localFactory := informers.NewSharedInformerFactory(localClient, 10*time.Hour) + remoteFactory := informers.NewSharedInformerFactory(remoteClient, 10*time.Hour) + cfg := offloadingv1beta1.ReflectorConfig{ + NumWorkers: 0, + Type: offloadingv1beta1.CustomLiqo, + } + rfl := dra.NewResourceClaimReflector(&cfg).(*dra.ResourceClaimReflector) + rfl.Start(ctx, options.New(localClient, localFactory.Core().V1().Pods())) + reflector = rfl.NewNamespaced(options.NewNamespaced(). + WithLocal(LocalNamespace, localClient, localFactory). + WithRemote(RemoteNamespace, remoteClient, remoteFactory). + WithHandlerFactory(FakeEventHandler). + WithEventBroadcaster(record.NewBroadcaster()). + WithReflectionType(offloadingv1beta1.CustomLiqo). + WithForgingOpts(testutil.FakeForgingOpts())) + + localFactory.Start(ctx.Done()) + remoteFactory.Start(ctx.Done()) + localFactory.WaitForCacheSync(ctx.Done()) + remoteFactory.WaitForCacheSync(ctx.Done()) + + err = reflector.Handle(trace.ContextWithTrace(ctx, trace.New("ResourceClaim")), claimName) + }) + + Describe("Handle", func() { + Context("when both local and remote claims are missing", func() { + It("should be a no-op and return nil", func() { + Expect(err).ToNot(HaveOccurred()) + _, gerr := remoteClient.ResourceV1().ResourceClaims(RemoteNamespace).Get(ctx, claimName, metav1.GetOptions{}) + Expect(kerrors.IsNotFound(gerr)).To(BeTrue()) + }) + }) + + Context("when the local exists and the remote does not", func() { + Context("with a referenced DeviceClass that exists locally and is missing remotely", func() { + BeforeEach(func() { + createLocalDeviceClass(deviceClassName) + createLocalClaim(mkBasicLocalClaim()) + }) + + It("should reflect the DeviceClass to remote with reflection labels and create the remote claim", func() { + Expect(err).ToNot(HaveOccurred()) + + dc, gerr := remoteClient.ResourceV1().DeviceClasses().Get(ctx, deviceClassName, metav1.GetOptions{}) + Expect(gerr).ToNot(HaveOccurred()) + Expect(forge.IsReflected(dc)).To(BeTrue()) + + rc, gerr := remoteClient.ResourceV1().ResourceClaims(RemoteNamespace).Get(ctx, claimName, metav1.GetOptions{}) + Expect(gerr).ToNot(HaveOccurred()) + Expect(rc.Labels).To(HaveKeyWithValue(fooVal, barVal)) + Expect(rc.Labels).ToNot(HaveKey(testutil.FakeNotReflectedLabelKey)) + Expect(rc.Labels).To(HaveKeyWithValue(forge.LiqoOriginClusterIDKey, LocalClusterID)) + Expect(rc.Labels).To(HaveKeyWithValue(forge.LiqoDestinationClusterIDKey, RemoteClusterID)) + }) + }) + + Context("with a referenced DeviceClass that is missing locally", func() { + BeforeEach(func() { createLocalClaim(mkBasicLocalClaim()) }) + + It("should skip the claim (no remote claim created) and not return an error", func() { + // Implementation translates the local-not-found to a soft skip + warning. + Expect(err).ToNot(HaveOccurred()) + _, gerr := remoteClient.ResourceV1().ResourceClaims(RemoteNamespace).Get(ctx, claimName, metav1.GetOptions{}) + Expect(kerrors.IsNotFound(gerr)).To(BeTrue()) + }) + }) + + Context("with the referenced DeviceClass already present remotely", func() { + var initialDCResourceVersion string + + BeforeEach(func() { + createLocalDeviceClass(deviceClassName) + existing := createRemoteDeviceClass(deviceClassName) + initialDCResourceVersion = existing.ResourceVersion + createLocalClaim(mkBasicLocalClaim()) + }) + + It("should not re-create the DeviceClass remotely (resourceVersion preserved)", func() { + Expect(err).ToNot(HaveOccurred()) + dc, gerr := remoteClient.ResourceV1().DeviceClasses().Get(ctx, deviceClassName, metav1.GetOptions{}) + Expect(gerr).ToNot(HaveOccurred()) + Expect(dc.ResourceVersion).To(Equal(initialDCResourceVersion)) + }) + }) + + Context("with multiple DeviceClasses across Exactly and FirstAvailable", func() { + const dc2 = "fpga-class" + + BeforeEach(func() { + createLocalDeviceClass(deviceClassName) + createLocalDeviceClass(dc2) + l := mkBasicLocalClaim() + l.Spec.Devices.Requests = append(l.Spec.Devices.Requests, resourcev1.DeviceRequest{ + Name: "req-2", + FirstAvailable: []resourcev1.DeviceSubRequest{ + {Name: "s1", DeviceClassName: dc2}, + {Name: "s2", DeviceClassName: deviceClassName}, // duplicate + }, + }) + createLocalClaim(l) + }) + + It("should reflect each unique DeviceClass exactly once on the remote", func() { + Expect(err).ToNot(HaveOccurred()) + _, gerr := remoteClient.ResourceV1().DeviceClasses().Get(ctx, deviceClassName, metav1.GetOptions{}) + Expect(gerr).ToNot(HaveOccurred()) + _, gerr = remoteClient.ResourceV1().DeviceClasses().Get(ctx, dc2, metav1.GetOptions{}) + Expect(gerr).ToNot(HaveOccurred()) + }) + }) + }) + + Context("when the local exists and the remote already exists and is reflected", func() { + Context("with the same labels and annotations as desired", func() { + var rvBefore string + + BeforeEach(func() { + createLocalDeviceClass(deviceClassName) + createRemoteDeviceClass(deviceClassName) + local := createLocalClaim(mkBasicLocalClaim()) + + desired := forge.RemoteResourceClaim(local, RemoteNamespace, + testutil.FakeForgingOpts().LabelsNotReflected, + testutil.FakeForgingOpts().AnnotationsNotReflected) + created := createRemoteClaim(desired) + rvBefore = created.ResourceVersion + }) + + It("should be a no-op (resourceVersion unchanged)", func() { + Expect(err).ToNot(HaveOccurred()) + rc, gerr := remoteClient.ResourceV1().ResourceClaims(RemoteNamespace).Get(ctx, claimName, metav1.GetOptions{}) + Expect(gerr).ToNot(HaveOccurred()) + Expect(rc.ResourceVersion).To(Equal(rvBefore)) + }) + }) + + Context("with stale labels", func() { + BeforeEach(func() { + createLocalDeviceClass(deviceClassName) + createRemoteDeviceClass(deviceClassName) + createLocalClaim(mkBasicLocalClaim()) + + stale := &resourcev1.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Labels: map[string]string{ + forge.LiqoOriginClusterIDKey: LocalClusterID, + forge.LiqoDestinationClusterIDKey: RemoteClusterID, + "old": "label", + }, + }, + Spec: resourcev1.ResourceClaimSpec{ + Devices: resourcev1.DeviceClaim{ + Requests: []resourcev1.DeviceRequest{{ + Name: fakeClaimName, + Exactly: &resourcev1.ExactDeviceRequest{DeviceClassName: deviceClassName, Count: 1}, + }}, + }, + }, + } + createRemoteClaim(stale) + }) + + It("should update labels on the remote claim", func() { + Expect(err).ToNot(HaveOccurred()) + rc, gerr := remoteClient.ResourceV1().ResourceClaims(RemoteNamespace).Get(ctx, claimName, metav1.GetOptions{}) + Expect(gerr).ToNot(HaveOccurred()) + Expect(rc.Labels).To(HaveKeyWithValue(fooVal, barVal)) + Expect(rc.Labels).ToNot(HaveKey("old")) + }) + }) + }) + + Context("when a non-reflected remote claim exists", func() { + var rvBefore string + + BeforeEach(func() { + createLocalDeviceClass(deviceClassName) + createLocalClaim(mkBasicLocalClaim()) + existing := &resourcev1.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{Name: claimName}, + Spec: resourcev1.ResourceClaimSpec{ + Devices: resourcev1.DeviceClaim{ + Requests: []resourcev1.DeviceRequest{{ + Name: fakeClaimName, + Exactly: &resourcev1.ExactDeviceRequest{DeviceClassName: deviceClassName, Count: 1}, + }}, + }, + }, + } + created := createRemoteClaim(existing) + rvBefore = created.ResourceVersion + }) + + It("should skip without modifying the remote claim", func() { + Expect(err).ToNot(HaveOccurred()) + rc, gerr := remoteClient.ResourceV1().ResourceClaims(RemoteNamespace).Get(ctx, claimName, metav1.GetOptions{}) + Expect(gerr).ToNot(HaveOccurred()) + Expect(rc.ResourceVersion).To(Equal(rvBefore)) + }) + }) + + Context("when the local is missing and the remote exists", func() { + Context("and the remote is reflected by Liqo", func() { + BeforeEach(func() { + createRemoteClaim(&resourcev1.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Labels: forge.ReflectionLabels(), + }, + Spec: resourcev1.ResourceClaimSpec{ + Devices: resourcev1.DeviceClaim{ + Requests: []resourcev1.DeviceRequest{{ + Name: fakeClaimName, + Exactly: &resourcev1.ExactDeviceRequest{DeviceClassName: deviceClassName, Count: 1}, + }}, + }, + }, + }) + }) + + It("should delete the remote claim", func() { + Expect(err).ToNot(HaveOccurred()) + _, gerr := remoteClient.ResourceV1().ResourceClaims(RemoteNamespace).Get(ctx, claimName, metav1.GetOptions{}) + Expect(kerrors.IsNotFound(gerr)).To(BeTrue()) + }) + }) + + Context("and the remote is NOT reflected by Liqo", func() { + var rvBefore string + + BeforeEach(func() { + created := createRemoteClaim(&resourcev1.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{Name: claimName}, + Spec: resourcev1.ResourceClaimSpec{ + Devices: resourcev1.DeviceClaim{ + Requests: []resourcev1.DeviceRequest{{ + Name: fakeClaimName, + Exactly: &resourcev1.ExactDeviceRequest{DeviceClassName: deviceClassName, Count: 1}, + }}, + }, + }, + }) + rvBefore = created.ResourceVersion + }) + + It("should leave the remote claim untouched", func() { + Expect(err).ToNot(HaveOccurred()) + rc, gerr := remoteClient.ResourceV1().ResourceClaims(RemoteNamespace).Get(ctx, claimName, metav1.GetOptions{}) + Expect(gerr).ToNot(HaveOccurred()) + Expect(rc.ResourceVersion).To(Equal(rvBefore)) + }) + }) + }) + }) +}) diff --git a/pkg/virtualKubelet/reflection/dra/resourceslice.go b/pkg/virtualKubelet/reflection/dra/resourceslice.go new file mode 100644 index 0000000000..a50d7d9a0f --- /dev/null +++ b/pkg/virtualKubelet/reflection/dra/resourceslice.go @@ -0,0 +1,345 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dra + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + corev1listers "k8s.io/client-go/listers/core/v1" + resourcev1listers "k8s.io/client-go/listers/resource/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + offloadingv1beta1 "github.com/liqotech/liqo/apis/offloading/v1beta1" + "github.com/liqotech/liqo/pkg/consts" + "github.com/liqotech/liqo/pkg/leaderelection" + "github.com/liqotech/liqo/pkg/virtualKubelet/forge" + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/generic" + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/manager" + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/options" +) + +const ( + // ResourceSliceReflectorName is the name associated with the ResourceSlice reflector. + ResourceSliceReflectorName = "ResourceSlice" +) + +var _ manager.Reflector = (*ResourceSliceReflector)(nil) + +// ResourceSliceReflector is a cluster-scoped, leader-only reflector that mirrors +// ResourceSlice objects from the remote cluster into the local cluster. The leader VK +// across the peering handles slices for every node in the peering, not just its own. +type ResourceSliceReflector struct { + name string + workers uint + + localClient kubernetes.Interface + remoteClient kubernetes.Interface + resync time.Duration + + workqueue workqueue.TypedRateLimitingInterface[types.NamespacedName] + + // Listers populated in Start. + localSlices resourcev1listers.ResourceSliceLister + remoteSlices resourcev1listers.ResourceSliceLister + localNodes corev1listers.NodeLister + + forgingOpts *forge.ForgingOpts +} + +// NewResourceSliceReflector returns a new ResourceSliceReflector. When the configured +// number of workers is zero, a dummy reflector is returned that performs no work. +func NewResourceSliceReflector(localClient, remoteClient kubernetes.Interface, + resync time.Duration, reflectorConfig *offloadingv1beta1.ReflectorConfig) manager.Reflector { + if reflectorConfig.NumWorkers == 0 { + return generic.NewDummyReflector(ResourceSliceReflectorName) + } + + return &ResourceSliceReflector{ + name: ResourceSliceReflectorName, + workers: reflectorConfig.NumWorkers, + localClient: localClient, + remoteClient: remoteClient, + resync: resync, + workqueue: workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[types.NamespacedName](), + workqueue.TypedRateLimitingQueueConfig[types.NamespacedName]{Name: ResourceSliceReflectorName}), + } +} + +// String returns the name of the reflector. +func (rsr *ResourceSliceReflector) String() string { return rsr.name } + +// Start sets up the cluster-scoped local and remote informers and launches the workers. +// StartNamespace is a no-op for this reflector. +func (rsr *ResourceSliceReflector) Start(ctx context.Context, opts *options.ReflectorOpts) { + klog.Infof("Starting the %v reflector with %v workers (leader-only)", rsr.name, rsr.workers) + rsr.forgingOpts = opts.ForgingOpts + if rsr.forgingOpts == nil { + rsr.forgingOpts = &forge.ForgingOpts{} + } + + localFactory := informers.NewSharedInformerFactory(rsr.localClient, rsr.resync) + remoteFactory := informers.NewSharedInformerFactory(rsr.remoteClient, rsr.resync) + // Separate factory for nodes to just listen on the virtual nodes changes. + nodeFactory := informers.NewSharedInformerFactoryWithOptions(rsr.localClient, rsr.resync, + informers.WithTweakListOptions(func(opts *metav1.ListOptions) { + opts.LabelSelector = consts.TypeLabel + "=" + consts.TypeNode + }), + ) + + localSliceInformer := localFactory.Resource().V1().ResourceSlices() + remoteSliceInformer := remoteFactory.Resource().V1().ResourceSlices() + localNodeInformer := nodeFactory.Core().V1().Nodes() + + _, err := localSliceInformer.Informer().AddEventHandler(rsr.handlers()) + utilruntime.Must(err) + _, err = remoteSliceInformer.Informer().AddEventHandler(rsr.handlers()) + utilruntime.Must(err) + // When a virtual node appears, enqueue the remote slices for that node, + // as they may have been skipped due to the missing local node. + _, err = localNodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { + node, ok := obj.(*corev1.Node) + if !ok { + return + } + rsr.enqueueRemoteSlicesForNode(node.Name) + }, + }) + utilruntime.Must(err) + + rsr.localSlices = localSliceInformer.Lister() + rsr.remoteSlices = remoteSliceInformer.Lister() + rsr.localNodes = localNodeInformer.Lister() + + localFactory.Start(ctx.Done()) + remoteFactory.Start(ctx.Done()) + nodeFactory.Start(ctx.Done()) + localFactory.WaitForCacheSync(ctx.Done()) + remoteFactory.WaitForCacheSync(ctx.Done()) + nodeFactory.WaitForCacheSync(ctx.Done()) + + for i := uint(0); i < rsr.workers; i++ { + go wait.Until(rsr.runWorker, time.Second, ctx.Done()) + } + + go func() { + <-ctx.Done() + rsr.workqueue.ShutDown() + }() +} + +// StartNamespace is a no-op: the reflector is cluster-scoped. +func (rsr *ResourceSliceReflector) StartNamespace(_ *options.NamespacedOpts) {} + +// StopNamespace is a no-op: the reflector is cluster-scoped. +func (rsr *ResourceSliceReflector) StopNamespace(_, _ string) {} + +// Resync re-enqueues every known remote and local slice so orphaned local +// slices (missed remote-delete events) also get reconciled. +func (rsr *ResourceSliceReflector) Resync() error { + if rsr.remoteSlices != nil { + remote, err := rsr.remoteSlices.List(labels.Everything()) + if err != nil { + klog.Errorf("Failed to list remote ResourceSlices for resync: %v", err) + } else { + for _, s := range remote { + rsr.workqueue.Add(types.NamespacedName{Name: s.Name}) + } + } + } + if rsr.localSlices != nil { + local, err := rsr.localSlices.List(labels.Everything()) + if err != nil { + klog.Errorf("Failed to list local ResourceSlices for resync: %v", err) + } else { + for _, s := range local { + rsr.workqueue.Add(types.NamespacedName{Name: s.Name}) + } + } + } + return nil +} + +// enqueueRemoteSlicesForNode enqueues only the remote slices whose Spec.NodeName +// matches the given local virtual node name. +func (rsr *ResourceSliceReflector) enqueueRemoteSlicesForNode(nodeName string) { + if rsr.remoteSlices == nil { + return + } + slices, err := rsr.remoteSlices.List(labels.Everything()) + if err != nil { + klog.Errorf("Failed to list remote ResourceSlices: %v", err) + return + } + for _, s := range slices { + if s.Spec.NodeName != nil && *s.Spec.NodeName == nodeName { + rsr.workqueue.Add(types.NamespacedName{Name: s.Name}) + } + } +} + +func (rsr *ResourceSliceReflector) handlers() cache.ResourceEventHandler { + enqueue := func(ev watch.EventType, obj any) { + if ev == watch.Deleted { + switch t := obj.(type) { + case cache.DeletedFinalStateUnknown: + obj = t.Obj + case *cache.DeletedFinalStateUnknown: + obj = t.Obj + } + } + + md, err := meta.Accessor(obj) + if err != nil { + klog.Errorf("Failed metadata accessor in %v reflector: obj=%T err=%v", rsr.name, obj, err) + return + } + rsr.workqueue.Add(types.NamespacedName{Name: md.GetName()}) + } + return cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { enqueue(watch.Added, obj) }, + UpdateFunc: func(_, obj any) { enqueue(watch.Modified, obj) }, + DeleteFunc: func(obj any) { enqueue(watch.Deleted, obj) }, + } +} + +func (rsr *ResourceSliceReflector) runWorker() { + for rsr.processNext() { + } +} + +func (rsr *ResourceSliceReflector) processNext() bool { + key, shutdown := rsr.workqueue.Get() + if shutdown { + return false + } + defer rsr.workqueue.Done(key) + + if !leaderelection.IsLeader() { + klog.V(4).Infof("Skipping %v reflector item %v because the node is not the leader", rsr.name, key) + return true + } + + err := rsr.handle(context.Background(), key.Name) + if err != nil { + rsr.workqueue.AddRateLimited(key) + return true + } + + rsr.workqueue.Forget(key) + return true +} + +// handle reconciles a single ResourceSlice by name. +func (rsr *ResourceSliceReflector) handle(ctx context.Context, name string) error { + klog.V(4).Infof("Handling reflection of ResourceSlice %q", name) + + local, lerr := rsr.localSlices.Get(name) + if lerr != nil && !kerrors.IsNotFound(lerr) { + klog.Errorf("Failed to get local ResourceSlice %q: %v", name, lerr) + return fmt.Errorf("getting local ResourceSlice %q: %w", name, lerr) + } + + // Refuse to mutate a local slice we don't own. + if lerr == nil && !forge.IsReflected(local) { + klog.V(4).Infof("Skipping reflection of ResourceSlice %q: local exists and is not managed by us", name) + return nil + } + + remote, rerr := rsr.remoteSlices.Get(name) + if rerr != nil && !kerrors.IsNotFound(rerr) { + klog.Errorf("Failed to get remote ResourceSlice %q: %v", name, rerr) + return fmt.Errorf("getting remote ResourceSlice %q: %w", name, rerr) + } + + // Remote vanished -> ensure local is also gone. + if kerrors.IsNotFound(rerr) { + if local != nil { + klog.Infof("Deleting local ResourceSlice %q since remote no longer exists", name) + err := rsr.localClient.ResourceV1().ResourceSlices().Delete(ctx, name, + *metav1.NewPreconditionDeleteOptions(string(local.GetUID()))) + if err != nil && !kerrors.IsNotFound(err) { + return fmt.Errorf("deleting ResourceSlice %q: %w", name, err) + } + } + return nil + } + + // We currently only handle slices whose Spec.NodeName is set. + // TODO: support spec.NodeSelector, spec.AllNodes and spec.PerDeviceNodeSelection. + if remote.Spec.NodeName == nil || *remote.Spec.NodeName == "" { + klog.V(4).Infof("Skipping reflection of ResourceSlice %q: spec.NodeName is unset (NodeSelector/AllNodes not yet supported)", name) + return nil + } + + // Resolve the local virtual node by name. The 1:1 mapping in this fork means + // remote.Spec.NodeName == local virtual node name. + localNode, err := rsr.localNodes.Get(*remote.Spec.NodeName) + if err != nil { + if kerrors.IsNotFound(err) { + klog.V(4).Infof("Skipping reflection of ResourceSlice %q: no local node %q yet", name, *remote.Spec.NodeName) + return nil + } + return fmt.Errorf("getting local node %q: %w", *remote.Spec.NodeName, err) + } + + desired := forge.LocalResourceSlice(remote, localNode, rsr.forgingOpts.LabelsNotReflected, rsr.forgingOpts.AnnotationsNotReflected) + + if kerrors.IsNotFound(lerr) { + if _, err := rsr.localClient.ResourceV1().ResourceSlices().Create(ctx, desired, metav1.CreateOptions{}); err != nil { + klog.Errorf("Failed to create local ResourceSlice %q: %v", name, err) + return fmt.Errorf("creating local ResourceSlice %q: %w", name, err) + } + klog.Infof("Local ResourceSlice %q successfully created (owner node: %q)", name, localNode.Name) + return nil + } + + if apiequality.Semantic.DeepEqual(local.Labels, desired.Labels) && + apiequality.Semantic.DeepEqual(local.Annotations, desired.Annotations) && + apiequality.Semantic.DeepEqual(local.OwnerReferences, desired.OwnerReferences) && + apiequality.Semantic.DeepEqual(local.Spec, desired.Spec) { + klog.V(4).Infof("Local ResourceSlice %q is already up-to-date", name) + return nil + } + + updated := local.DeepCopy() + updated.Labels = desired.Labels + updated.Annotations = desired.Annotations + updated.OwnerReferences = desired.OwnerReferences + updated.Spec = desired.Spec + if _, err := rsr.localClient.ResourceV1().ResourceSlices().Update(ctx, updated, metav1.UpdateOptions{}); err != nil { + klog.Errorf("Failed to update local ResourceSlice %q: %v", name, err) + return fmt.Errorf("updating local ResourceSlice %q: %w", name, err) + } + klog.V(4).Infof("Local ResourceSlice %q successfully updated", name) + return nil +} diff --git a/pkg/virtualKubelet/reflection/dra/resourceslice_test.go b/pkg/virtualKubelet/reflection/dra/resourceslice_test.go new file mode 100644 index 0000000000..fc7e0f5d68 --- /dev/null +++ b/pkg/virtualKubelet/reflection/dra/resourceslice_test.go @@ -0,0 +1,387 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dra_test + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + resourcev1 "k8s.io/api/resource/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + corev1listers "k8s.io/client-go/listers/core/v1" + resourcev1listers "k8s.io/client-go/listers/resource/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/utils/ptr" + + "github.com/liqotech/liqo/pkg/utils/testutil" + "github.com/liqotech/liqo/pkg/virtualKubelet/forge" + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/dra" +) + +// errResourceSliceLister is a stub ResourceSliceLister whose Get always returns +// the configured error. Used to test error propagation paths that envtest can't +// reproduce via reactors (informer-backed listers swallow API errors). +type errResourceSliceLister struct { + err error +} + +func (e *errResourceSliceLister) List(_ labels.Selector) ([]*resourcev1.ResourceSlice, error) { + return nil, e.err +} +func (e *errResourceSliceLister) Get(_ string) (*resourcev1.ResourceSlice, error) { + return nil, e.err +} + +func newSliceIndexer() cache.Indexer { + return cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) +} + +func newNodeLister(nodes ...*corev1.Node) corev1listers.NodeLister { + idx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + for _, n := range nodes { + Expect(idx.Add(n)).To(Succeed()) + } + return corev1listers.NewNodeLister(idx) +} + +var _ = Describe("ResourceSliceReflector", func() { + const sliceName = "slice-1" + + var ( + reflector *dra.ResourceSliceReflector + localSlicesIndex cache.Indexer + remoteSlicesIndex cache.Indexer + localNodes corev1listers.NodeLister + opts *forge.ForgingOpts + ) + + // localNode mirrors the suite-created virtual node so OwnerReference + // assertions can compare against LiqoNodeUID. + mkLocalNode := func() *corev1.Node { + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: LiqoNodeName, UID: LiqoNodeUID}, + } + } + + BeforeEach(func() { + opts = testutil.FakeForgingOpts() + localSlicesIndex = newSliceIndexer() + remoteSlicesIndex = newSliceIndexer() + localNodes = newNodeLister(mkLocalNode()) + }) + + JustBeforeEach(func() { + reflector = dra.NewResourceSliceReflectorForTest( + localClient, remoteClient, + resourcev1listers.NewResourceSliceLister(localSlicesIndex), + resourcev1listers.NewResourceSliceLister(remoteSlicesIndex), + localNodes, + opts, + ) + }) + + AfterEach(func() { + // Best-effort cleanup of any slice the test may have created on the envtest API. + _ = localClient.ResourceV1().ResourceSlices().Delete(ctx, sliceName, metav1.DeleteOptions{}) + }) + + Describe("handle", func() { + Context("when both local and remote are missing", func() { + It("should be a no-op and return nil", func() { + Expect(dra.Handle(reflector, ctx, sliceName)).To(Succeed()) + _, err := localClient.ResourceV1().ResourceSlices().Get(ctx, sliceName, metav1.GetOptions{}) + Expect(kerrors.IsNotFound(err)).To(BeTrue()) + }) + }) + + Context("when the remote vanished", func() { + When("the local slice exists and is reflected by Liqo", func() { + BeforeEach(func() { + created, err := localClient.ResourceV1().ResourceSlices().Create(ctx, &resourcev1.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: sliceName, + Labels: forge.ReflectionLabels(), + }, + Spec: resourcev1.ResourceSliceSpec{ + Driver: fakeDriverName, + Pool: resourcev1.ResourcePool{Name: fakePoolName, ResourceSliceCount: 1}, + NodeName: ptr.To(LiqoNodeName), + }, + }, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(localSlicesIndex.Add(created)).To(Succeed()) + }) + + It("should delete the local slice", func() { + Expect(dra.Handle(reflector, ctx, sliceName)).To(Succeed()) + _, err := localClient.ResourceV1().ResourceSlices().Get(ctx, sliceName, metav1.GetOptions{}) + Expect(kerrors.IsNotFound(err)).To(BeTrue()) + }) + }) + + When("the local slice exists but is not reflected by Liqo", func() { + BeforeEach(func() { + created, err := localClient.ResourceV1().ResourceSlices().Create(ctx, &resourcev1.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{Name: sliceName}, + Spec: resourcev1.ResourceSliceSpec{ + Driver: fakeDriverName, + Pool: resourcev1.ResourcePool{Name: fakePoolName, ResourceSliceCount: 1}, + NodeName: ptr.To(LiqoNodeName), + }, + }, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(localSlicesIndex.Add(created)).To(Succeed()) + }) + + It("should not delete the local slice", func() { + Expect(dra.Handle(reflector, ctx, sliceName)).To(Succeed()) + _, err := localClient.ResourceV1().ResourceSlices().Get(ctx, sliceName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + }) + }) + }) + + Context("when the remote slice exists", func() { + var remote *resourcev1.ResourceSlice + + BeforeEach(func() { + remote = &resourcev1.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: sliceName, + Labels: map[string]string{ + "foo": barVal, + testutil.FakeNotReflectedLabelKey: trueVal, + }, + }, + Spec: resourcev1.ResourceSliceSpec{ + Driver: fakeDriverName, + Pool: resourcev1.ResourcePool{Name: fakePoolName, ResourceSliceCount: 1}, + NodeName: ptr.To(LiqoNodeName), + Devices: []resourcev1.Device{{Name: "dev-1"}}, + }, + } + }) + + JustBeforeEach(func() { Expect(remoteSlicesIndex.Add(remote)).To(Succeed()) }) + + When("the remote has no NodeName", func() { + BeforeEach(func() { remote.Spec.NodeName = nil }) + It("should be a no-op", func() { + Expect(dra.Handle(reflector, ctx, sliceName)).To(Succeed()) + _, err := localClient.ResourceV1().ResourceSlices().Get(ctx, sliceName, metav1.GetOptions{}) + Expect(kerrors.IsNotFound(err)).To(BeTrue()) + }) + }) + + When("the remote has an empty-string NodeName", func() { + BeforeEach(func() { remote.Spec.NodeName = ptr.To("") }) + It("should be a no-op", func() { + Expect(dra.Handle(reflector, ctx, sliceName)).To(Succeed()) + _, err := localClient.ResourceV1().ResourceSlices().Get(ctx, sliceName, metav1.GetOptions{}) + Expect(kerrors.IsNotFound(err)).To(BeTrue()) + }) + }) + + When("the local virtual node is not in cache", func() { + BeforeEach(func() { + localNodes = newNodeLister() + }) + It("should return nil and not create the local slice", func() { + Expect(dra.Handle(reflector, ctx, sliceName)).To(Succeed()) + _, err := localClient.ResourceV1().ResourceSlices().Get(ctx, sliceName, metav1.GetOptions{}) + Expect(kerrors.IsNotFound(err)).To(BeTrue()) + }) + }) + + When("the local slice does not yet exist", func() { + It("should create the local slice with reflection labels and OwnerReference to the local node", func() { + Expect(dra.Handle(reflector, ctx, sliceName)).To(Succeed()) + + got, err := localClient.ResourceV1().ResourceSlices().Get(ctx, sliceName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Labels).To(HaveKeyWithValue(fooVal, barVal)) + Expect(got.Labels).ToNot(HaveKey(testutil.FakeNotReflectedLabelKey)) + Expect(got.Labels).To(HaveKeyWithValue(forge.LiqoOriginClusterIDKey, LocalClusterID)) + Expect(got.OwnerReferences).To(HaveLen(1)) + Expect(got.OwnerReferences[0].Name).To(Equal(LiqoNodeName)) + Expect(got.OwnerReferences[0].UID).To(Equal(LiqoNodeUID)) + Expect(got.Spec.Driver).To(Equal(fakeDriverName)) + }) + }) + + When("the local slice exists, is reflected, and is already up-to-date", func() { + var rvBefore string + + BeforeEach(func() { + desired := forge.LocalResourceSlice(remote, mkLocalNode(), + opts.LabelsNotReflected, opts.AnnotationsNotReflected) + created, err := localClient.ResourceV1().ResourceSlices().Create(ctx, desired, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + rvBefore = created.ResourceVersion + Expect(localSlicesIndex.Add(created)).To(Succeed()) + }) + + It("should not issue an Update (resourceVersion unchanged)", func() { + Expect(dra.Handle(reflector, ctx, sliceName)).To(Succeed()) + got, err := localClient.ResourceV1().ResourceSlices().Get(ctx, sliceName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(got.ResourceVersion).To(Equal(rvBefore)) + }) + }) + + When("the local slice exists, is reflected, but has stale labels", func() { + BeforeEach(func() { + stale := forge.LocalResourceSlice(remote, mkLocalNode(), + opts.LabelsNotReflected, opts.AnnotationsNotReflected) + stale.Labels["stale"] = "yes" + created, err := localClient.ResourceV1().ResourceSlices().Create(ctx, stale, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(localSlicesIndex.Add(created)).To(Succeed()) + }) + + It("should issue an Update that reconciles labels to match the desired state", func() { + Expect(dra.Handle(reflector, ctx, sliceName)).To(Succeed()) + got, err := localClient.ResourceV1().ResourceSlices().Get(ctx, sliceName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Labels).ToNot(HaveKey("stale")) + Expect(got.Labels).To(HaveKeyWithValue(fooVal, barVal)) + Expect(got.Labels).To(HaveKeyWithValue(forge.LiqoOriginClusterIDKey, LocalClusterID)) + }) + }) + + When("the local slice exists but is not reflected by Liqo", func() { + var rvBefore string + + BeforeEach(func() { + created, err := localClient.ResourceV1().ResourceSlices().Create(ctx, &resourcev1.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{Name: sliceName}, + Spec: resourcev1.ResourceSliceSpec{ + Driver: "other", + Pool: resourcev1.ResourcePool{Name: fakePoolName, ResourceSliceCount: 1}, + NodeName: ptr.To(LiqoNodeName), + }, + }, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + rvBefore = created.ResourceVersion + Expect(localSlicesIndex.Add(created)).To(Succeed()) + }) + + It("should skip without modifying the local slice", func() { + Expect(dra.Handle(reflector, ctx, sliceName)).To(Succeed()) + got, err := localClient.ResourceV1().ResourceSlices().Get(ctx, sliceName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(got.ResourceVersion).To(Equal(rvBefore)) + }) + }) + }) + + Context("when the local lister returns a non-NotFound error", func() { + JustBeforeEach(func() { + reflector = dra.NewResourceSliceReflectorForTest( + localClient, remoteClient, + &errResourceSliceLister{err: errors.New("local boom")}, + resourcev1listers.NewResourceSliceLister(remoteSlicesIndex), + localNodes, + opts, + ) + }) + + It("should propagate the error", func() { + err := dra.Handle(reflector, ctx, sliceName) + Expect(err).To(MatchError(ContainSubstring("local boom"))) + }) + }) + + Context("when the remote lister returns a non-NotFound error", func() { + JustBeforeEach(func() { + reflector = dra.NewResourceSliceReflectorForTest( + localClient, remoteClient, + resourcev1listers.NewResourceSliceLister(localSlicesIndex), + &errResourceSliceLister{err: errors.New("remote boom")}, + localNodes, + opts, + ) + }) + + It("should propagate the error", func() { + err := dra.Handle(reflector, ctx, sliceName) + Expect(err).To(MatchError(ContainSubstring("remote boom"))) + }) + }) + }) + + Describe("enqueueRemoteSlicesForNode", func() { + mkSlice := func(name, nodeName string) *resourcev1.ResourceSlice { + s := &resourcev1.ResourceSlice{ObjectMeta: metav1.ObjectMeta{Name: name}} + if nodeName != "" { + s.Spec.NodeName = ptr.To(nodeName) + } + return s + } + + It("should only enqueue slices whose NodeName matches", func() { + Expect(remoteSlicesIndex.Add(mkSlice("a", LiqoNodeName))).To(Succeed()) + Expect(remoteSlicesIndex.Add(mkSlice("b", "other-node"))).To(Succeed()) + Expect(remoteSlicesIndex.Add(mkSlice("c", LiqoNodeName))).To(Succeed()) + + before := reflector.WorkqueueLen() + dra.EnqueueRemoteSlicesForNode(reflector, LiqoNodeName) + added := reflector.WorkqueueLen() - before + Expect(added).To(Equal(2)) + }) + + It("should skip slices with nil NodeName", func() { + Expect(remoteSlicesIndex.Add(mkSlice("a", ""))).To(Succeed()) + + before := reflector.WorkqueueLen() + dra.EnqueueRemoteSlicesForNode(reflector, LiqoNodeName) + Expect(reflector.WorkqueueLen() - before).To(Equal(0)) + }) + + It("should not panic on an empty cache", func() { + Expect(func() { dra.EnqueueRemoteSlicesForNode(reflector, LiqoNodeName) }).ToNot(Panic()) + }) + }) + + Describe("Resync", func() { + It("should add nothing to the workqueue when there are no slices", func() { + before := reflector.WorkqueueLen() + Expect(reflector.Resync()).To(Succeed()) + Expect(reflector.WorkqueueLen() - before).To(Equal(0)) + }) + + It("should enqueue every remote slice when only remotes exist", func() { + Expect(remoteSlicesIndex.Add(&resourcev1.ResourceSlice{ObjectMeta: metav1.ObjectMeta{Name: "a"}})).To(Succeed()) + Expect(remoteSlicesIndex.Add(&resourcev1.ResourceSlice{ObjectMeta: metav1.ObjectMeta{Name: "b"}})).To(Succeed()) + + before := reflector.WorkqueueLen() + Expect(reflector.Resync()).To(Succeed()) + Expect(reflector.WorkqueueLen() - before).To(Equal(2)) + }) + + It("should enqueue every local slice when only locals exist", func() { + Expect(localSlicesIndex.Add(&resourcev1.ResourceSlice{ObjectMeta: metav1.ObjectMeta{Name: "a"}})).To(Succeed()) + Expect(localSlicesIndex.Add(&resourcev1.ResourceSlice{ObjectMeta: metav1.ObjectMeta{Name: "b"}})).To(Succeed()) + + before := reflector.WorkqueueLen() + Expect(reflector.Resync()).To(Succeed()) + Expect(reflector.WorkqueueLen() - before).To(Equal(2)) + }) + }) +}) diff --git a/pkg/virtualKubelet/reflection/generic/reflector.go b/pkg/virtualKubelet/reflection/generic/reflector.go index de5fbfee23..332013ad8c 100644 --- a/pkg/virtualKubelet/reflection/generic/reflector.go +++ b/pkg/virtualKubelet/reflection/generic/reflector.go @@ -398,6 +398,12 @@ func WithoutFallback() FallbackReflectorFactoryFunc { return func(ro *options.ReflectorOpts) manager.FallbackReflector { return nil } } +// NewDummyReflector returns a dummy reflector that can be used when no workers are specified, to avoid starting the working queue and +// registering the informers. +func NewDummyReflector(name string) manager.Reflector { + return &dummyreflector{name: name} +} + type dummyreflector struct{ name string } // String returns the name of the dummy reflector. diff --git a/pkg/virtualKubelet/reflection/manager/manager.go b/pkg/virtualKubelet/reflection/manager/manager.go index 79af0c4d9c..7959966869 100644 --- a/pkg/virtualKubelet/reflection/manager/manager.go +++ b/pkg/virtualKubelet/reflection/manager/manager.go @@ -116,7 +116,9 @@ func (m *manager) Start(ctx context.Context) { ready := false for _, reflector := range m.reflectors { opts := options.New(m.local, m.localPodInformerFactory.Core().V1().Pods()). - WithReadinessFunc(func() bool { return ready }).WithEventBroadcaster(m.eventBroadcaster) + WithReadinessFunc(func() bool { return ready }). + WithEventBroadcaster(m.eventBroadcaster). + WithForgingOpts(&m.forgingOpts) reflector.Start(ctx, opts) } diff --git a/pkg/virtualKubelet/reflection/options/options.go b/pkg/virtualKubelet/reflection/options/options.go index c07d54ecaf..112e663428 100644 --- a/pkg/virtualKubelet/reflection/options/options.go +++ b/pkg/virtualKubelet/reflection/options/options.go @@ -41,6 +41,7 @@ type ReflectorOpts struct { LocalClient kubernetes.Interface LocalPodInformer corev1informers.PodInformer EventBroadcaster record.EventBroadcaster + ForgingOpts *forge.ForgingOpts HandlerFactory func(Keyer, ...EventFilter) cache.ResourceEventHandler @@ -64,12 +65,18 @@ func (ro *ReflectorOpts) WithReadinessFunc(ready func() bool) *ReflectorOpts { return ro } -// WithEventBroadcaster configures the event broadcaster of the NamespacedOpts. +// WithEventBroadcaster configures the event broadcaster of the ReflectorOpts. func (ro *ReflectorOpts) WithEventBroadcaster(broadcaster record.EventBroadcaster) *ReflectorOpts { ro.EventBroadcaster = broadcaster return ro } +// WithForgingOpts configures the reflection options of the ReflectorOpts. +func (ro *ReflectorOpts) WithForgingOpts(opts *forge.ForgingOpts) *ReflectorOpts { + ro.ForgingOpts = opts + return ro +} + // NamespacedOpts is a structure grouping the parameters to start a NamespacedReflector. type NamespacedOpts struct { LocalNamespace string diff --git a/pkg/virtualKubelet/reflection/resources/resources.go b/pkg/virtualKubelet/reflection/resources/resources.go index f97bf8e143..afd0604636 100644 --- a/pkg/virtualKubelet/reflection/resources/resources.go +++ b/pkg/virtualKubelet/reflection/resources/resources.go @@ -27,11 +27,17 @@ const ( Secret ResourceReflected = "secret" ServiceAccount ResourceReflected = "serviceaccount" PersistentVolumeClaim ResourceReflected = "persistentvolumeclaim" + LocalStoragePVC ResourceReflected = "localstoragepvc" Event ResourceReflected = "event" + ResourceSlice ResourceReflected = "resourceslice" + ResourceClaim ResourceReflected = "resourceclaim" ) // Reflectors is the list of all resources that can be reflected. -var Reflectors = []ResourceReflected{Pod, Service, EndpointSlice, Ingress, ConfigMap, Secret, ServiceAccount, PersistentVolumeClaim, Event} +var Reflectors = []ResourceReflected{ + Pod, Service, EndpointSlice, Ingress, ConfigMap, Secret, ServiceAccount, PersistentVolumeClaim, Event, + LocalStoragePVC, ResourceSlice, ResourceClaim, +} // ReflectorsCustomizableType is the list of resources for which the reflection type can be customized. -var ReflectorsCustomizableType = []ResourceReflected{Service, Ingress, ConfigMap, Secret, Event} +var ReflectorsCustomizableType = []ResourceReflected{Service, Ingress, ConfigMap, Secret, LocalStoragePVC, Event} diff --git a/pkg/virtualKubelet/reflection/storage/controller.go b/pkg/virtualKubelet/reflection/storage/controller.go index 3b9811e6a6..f1f3f2fbcf 100644 --- a/pkg/virtualKubelet/reflection/storage/controller.go +++ b/pkg/virtualKubelet/reflection/storage/controller.go @@ -18,15 +18,19 @@ import ( "context" "errors" "fmt" + "strings" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ref "k8s.io/client-go/tools/reference" "k8s.io/klog/v2" "k8s.io/kubectl/pkg/scheme" "sigs.k8s.io/sig-storage-lib-external-provisioner/v7/controller" "sigs.k8s.io/sig-storage-lib-external-provisioner/v7/util" + + "github.com/liqotech/liqo/pkg/consts" ) const ( @@ -49,14 +53,21 @@ const ( // - the volume binding mode is WaitForFirstConsumer // - the selected node is the one the reflector is managing. func (npvcr *NamespacedPersistentVolumeClaimReflector) shouldProvision(claim *corev1.PersistentVolumeClaim) (bool, error) { - if claim.Spec.VolumeName != "" { + if claim.Annotations == nil { + return false, nil + } + + // If PVC should not provisioned on all the edge nodes, skip if volume is already bound. + shouldProvisionOnAllEdges := claim.Annotations[consts.ProvisionPVCOnAllEdgesAnnotationKey] == consts.ProvisionPVCOnAllEdgesAnnotationValue + if claim.Spec.VolumeName != "" && !shouldProvisionOnAllEdges { return false, nil } if provisioner, found := claim.Annotations[annStorageProvisioner]; found { if npvcr.knownProvisioner(provisioner) { claimClass := util.GetPersistentVolumeClaimClass(claim) - if claimClass != npvcr.virtualStorageClassName { + // We need to provision if storage class is liqo or omni one. + if claimClass != npvcr.virtualStorageClassName && !strings.HasPrefix(claimClass, consts.OmniStorageClassPrefix) { return false, nil } @@ -74,6 +85,15 @@ func (npvcr *NamespacedPersistentVolumeClaimReflector) shouldProvision(claim *co if selectedNode, ok := claim.Annotations[annSelectedNode]; ok && selectedNode != "" { return true, nil } + // For provision-on-all-edges PVCs, annSelectedNode is removed once the PVC is bound. + // Still trigger provisioning if the remote PVC is missing on this cluster. + if shouldProvisionOnAllEdges && claim.Spec.VolumeName != "" { + _, err := npvcr.remotePersistentVolumeClaims.Get(claim.Name) + if kerrors.IsNotFound(err) { + return true, nil + } + return false, err + } return false, nil } // do not provision if the binding mode is not WaitForFirstConsumer @@ -103,11 +123,14 @@ func (npvcr *NamespacedPersistentVolumeClaimReflector) provisionClaimOperation(c // the locks. Check that PV (with deterministic name) hasn't been provisioned // yet. pvName := "pvc-" + string(claim.UID) - _, err := npvcr.volumes.Get(pvName) - if err == nil { - // Volume has been already provisioned, nothing to do. - klog.V(4).Infof("Persistentvolume %q already exists, skipping", pvName) - return controller.ProvisioningFinished, nil + shouldProvisionOnAllEdges := claim.Annotations[consts.ProvisionPVCOnAllEdgesAnnotationKey] == consts.ProvisionPVCOnAllEdgesAnnotationValue + if !shouldProvisionOnAllEdges { + _, err := npvcr.volumes.Get(pvName) + if err == nil { + // Volume has been already provisioned, nothing to do. + klog.V(4).Infof("Persistentvolume %q already exists, skipping", pvName) + return controller.ProvisioningFinished, nil + } } // Prepare a claimRef to the claim early (to fail before a volume is @@ -176,6 +199,12 @@ func (npvcr *NamespacedPersistentVolumeClaimReflector) provisionClaimOperation(c volume.Spec.StorageClassName = claimClass _, err = npvcr.localPersistentVolumesClient.Create(ctx, volume, metav1.CreateOptions{}) + if kerrors.IsAlreadyExists(err) && shouldProvisionOnAllEdges { + // For "provision on all edges" PVCs another VK may have already created the local PV. + // The remote PVC was provisioned successfully on this cluster, so this is not an error. + klog.V(4).Infof("Persistentvolume %q already exists, created by another VK (provision-on-all-edges)", pvName) + err = nil + } return controller.ProvisioningFinished, err } diff --git a/pkg/virtualKubelet/reflection/storage/localstoragepvc.go b/pkg/virtualKubelet/reflection/storage/localstoragepvc.go new file mode 100644 index 0000000000..cec1492674 --- /dev/null +++ b/pkg/virtualKubelet/reflection/storage/localstoragepvc.go @@ -0,0 +1,460 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + corev1apply "k8s.io/client-go/applyconfigurations/core/v1" + corev1clients "k8s.io/client-go/kubernetes/typed/core/v1" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/klog/v2" + "k8s.io/utils/trace" + "sigs.k8s.io/controller-runtime/pkg/client" + + offloadingv1beta1 "github.com/liqotech/liqo/apis/offloading/v1beta1" + "github.com/liqotech/liqo/pkg/consts" + "github.com/liqotech/liqo/pkg/utils/maps" + "github.com/liqotech/liqo/pkg/utils/virtualkubelet" + "github.com/liqotech/liqo/pkg/virtualKubelet/forge" + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/generic" + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/manager" + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/options" +) + +const ( + // LocalStoragePVCReflectorName -> The name associated with the local-storage PersistentVolumeClaim reflector. + LocalStoragePVCReflectorName = "LocalStoragePVC" +) + +var _ manager.NamespacedReflector = (*NamespacedLocalStoragePVCReflector)(nil) + +// NamespacedLocalStoragePVCReflector manages the reflection of local-storage PersistentVolumeClaims +// (and their bound PersistentVolumes) for a given pair of local and remote namespaces. +type NamespacedLocalStoragePVCReflector struct { + generic.NamespacedReflector + + localPersistentVolumeClaims corev1listers.PersistentVolumeClaimNamespaceLister + remotePersistentVolumeClaims corev1listers.PersistentVolumeClaimNamespaceLister + localPersistentVolumes corev1listers.PersistentVolumeLister + remotePersistentVolumes corev1listers.PersistentVolumeLister + remotePersistentVolumesClaimsClient corev1clients.PersistentVolumeClaimInterface + remotePersistentVolumesClient corev1clients.PersistentVolumeInterface +} + +// NewLocalStoragePVCReflector returns a new LocalStoragePVCReflector instance. +func NewLocalStoragePVCReflector(reflectorConfig *offloadingv1beta1.ReflectorConfig) manager.Reflector { + return generic.NewReflector(LocalStoragePVCReflectorName, + NewNamespacedLocalStoragePVCReflector(), + generic.WithoutFallback(), reflectorConfig.NumWorkers, reflectorConfig.Type, generic.ConcurrencyModeLeader) +} + +// NewNamespacedLocalStoragePVCReflector returns a function generating NamespacedLocalStoragePVCReflector instances. +func NewNamespacedLocalStoragePVCReflector() func(*options.NamespacedOpts) manager.NamespacedReflector { + return func(opts *options.NamespacedOpts) manager.NamespacedReflector { + local := opts.LocalFactory.Core().V1().PersistentVolumeClaims() + remote := opts.RemoteFactory.Core().V1().PersistentVolumeClaims() + localVolumes := opts.LocalFactory.Core().V1().PersistentVolumes() + remoteVolumes := opts.RemoteFactory.Core().V1().PersistentVolumes() + + // Using opts.LocalNamespace for both event handlers so that the object will be put in the same workqueue + // no matter the cluster, hence it will be processed by the handle function in the same way. + _, err := local.Informer().AddEventHandler(opts.HandlerFactory(generic.NamespacedKeyer(opts.LocalNamespace))) + utilruntime.Must(err) + _, err = remote.Informer().AddEventHandler(opts.HandlerFactory(generic.NamespacedKeyer(opts.LocalNamespace))) + utilruntime.Must(err) + + return &NamespacedLocalStoragePVCReflector{ + NamespacedReflector: generic.NewNamespacedReflector(opts, LocalStoragePVCReflectorName), + + localPersistentVolumeClaims: local.Lister().PersistentVolumeClaims(opts.LocalNamespace), + remotePersistentVolumeClaims: remote.Lister().PersistentVolumeClaims(opts.RemoteNamespace), + localPersistentVolumes: localVolumes.Lister(), + remotePersistentVolumes: remoteVolumes.Lister(), + remotePersistentVolumesClaimsClient: opts.RemoteClient.CoreV1().PersistentVolumeClaims(opts.RemoteNamespace), + remotePersistentVolumesClient: opts.RemoteClient.CoreV1().PersistentVolumes(), + } + } +} + +// Handle reconciles local-storage PersistentVolumeClaim objects. +func (nls *NamespacedLocalStoragePVCReflector) Handle(ctx context.Context, name string) error { + tracer := trace.FromContext(ctx) + + klog.V(4).Infof("Handling reflection of local PersistentVolumeClaim %q (remote: %q)", nls.LocalRef(name), nls.RemoteRef(name)) + local, lerr := nls.localPersistentVolumeClaims.Get(name) + utilruntime.Must(client.IgnoreNotFound(lerr)) + remote, rerr := nls.remotePersistentVolumeClaims.Get(name) + utilruntime.Must(client.IgnoreNotFound(rerr)) + tracer.Step("Retrieved the local and remote objects") + + // Abort the reflection if the remote object is not managed by us. + if rerr == nil && !isLocalStorageReflectedByUs(remote) { + if lerr == nil { + klog.Infof("Skipping reflection of local PersistentVolumeClaim %q as remote already exists and is not managed by us", nls.LocalRef(name)) + nls.Event(local, corev1.EventTypeWarning, forge.EventFailedReflection, forge.EventFailedReflectionAlreadyExistsMsg()) + } + return nil + } + + // Abort the reflection if the local object has the "skip-reflection" annotation. + if !kerrors.IsNotFound(lerr) { + skipReflection, err := nls.ShouldSkipReflection(local) + if err != nil { + klog.Errorf("Failed to check whether local PersistentVolumeClaim %q should be reflected: %v", nls.LocalRef(name), err) + return err + } + if skipReflection { + if nls.GetReflectionType() == offloadingv1beta1.DenyList { + klog.Infof("Skipping reflection of local PersistentVolumeClaim %q as marked with the skip annotation", nls.LocalRef(name)) + } else { // AllowList + klog.Infof("Skipping reflection of local PersistentVolumeClaim %q as not marked with the allow annotation", nls.LocalRef(name)) + } + nls.Event(local, corev1.EventTypeNormal, forge.EventReflectionDisabled, forge.EventObjectReflectionDisabledMsg(nls.GetReflectionType())) + if kerrors.IsNotFound(rerr) { // The remote object does not already exist, hence no further action is required. + return nil + } + + // Otherwise, let pretend the local object does not exist, so that the remote one gets deleted. + lerr = kerrors.NewNotFound(corev1.Resource("persistentvolumeclaim"), local.GetName()) + } + } + + tracer.Step("Performed the sanity checks") + + // The local PersistentVolumeClaim no longer exists. Ensure it is also absent from the remote cluster. + if kerrors.IsNotFound(lerr) { + defer tracer.Step("Ensured the absence of the remote object") + if !kerrors.IsNotFound(rerr) { + klog.V(4).Infof("Deleting remote PersistentVolumeClaim %q, since local %q no longer exists", nls.RemoteRef(name), nls.LocalRef(name)) + // Delete the PVC first to release the PV protection finalizer on the bound volume. + remotePVName := remote.Spec.VolumeName + if err := nls.DeleteRemote(ctx, nls.remotePersistentVolumesClaimsClient, LocalStoragePVCReflectorName, name, remote.GetUID()); err != nil { + return err + } + // Also clean up the reflected PV if present. + if remotePVName != "" { + if err := nls.deleteRemotePV(ctx, remotePVName); err != nil { + return err + } + } + return nil + } + + klog.V(4).Infof("Local PersistentVolumeClaim %q and remote PersistentVolumeClaim %q both vanished", nls.LocalRef(name), nls.RemoteRef(name)) + return nil + } + + // DeepCopy the local object to allow modifications. + local = local.DeepCopy() + + // Only reflect PVCs that are already bound to a PV. + if local.Spec.VolumeName == "" { + klog.V(4).Infof("Skipping reflection of unbound PersistentVolumeClaim %q", nls.LocalRef(name)) + return nil + } + tracer.Step("Confirmed the PVC is bound to a PV") + + // Retrieve the bound PV. + localPV, err := nls.localPersistentVolumes.Get(local.Spec.VolumeName) + if err != nil { + klog.Errorf("Error retrieving bound PersistentVolume %q for local PersistentVolumeClaim %q: %v", + local.Spec.VolumeName, nls.LocalRef(name), err.Error()) + nls.Event(local, corev1.EventTypeWarning, forge.EventFailedReflection, forge.EventFailedReflectionMsg(err)) + return err + } + tracer.Step("Retrieved the bound PersistentVolume") + + // Only handle PVCs backed by a local PersistentVolume. + if !isLocalStoragePV(localPV) { + klog.V(4).Infof("Skipping PersistentVolumeClaim %q since it is not bound to a local PersistentVolume", nls.LocalRef(name)) + return nil + } + tracer.Step("Confirmed the PVC is bound to a local PersistentVolume") + + // Forge and apply the remote PVC. + remotePVName := remotePVNameFromLocal(localPV) + remotePVCApply := forgeRemotePVC(local, nls.RemoteNamespace(), remotePVName, nls.ForgingOpts) + remotePVC, err := nls.remotePersistentVolumesClaimsClient.Apply(ctx, remotePVCApply, forge.ApplyOptions()) + if err != nil { + klog.Errorf("Error applying remote PersistentVolumeClaim %q: %v", nls.RemoteRef(name), err.Error()) + nls.Event(local, corev1.EventTypeWarning, forge.EventFailedReflection, forge.EventFailedReflectionMsg(err)) + return err + } + tracer.Step("Applied the remote PersistentVolumeClaim") + + // Forge and apply the remote PV, bound to the remote PVC. + remotePVApply := forgeRemotePV(localPV, remotePVC, nls.RemoteNamespace(), nls.ForgingOpts) + if _, err = nls.remotePersistentVolumesClient.Apply(ctx, remotePVApply, forge.ApplyOptions()); err != nil { + klog.Errorf("Error applying remote PersistentVolume %q: %v", remotePVName, err.Error()) + nls.Event(local, corev1.EventTypeWarning, forge.EventFailedReflection, forge.EventFailedReflectionMsg(err)) + return err + } + tracer.Step("Applied the remote PersistentVolume") + + nls.Event(local, corev1.EventTypeNormal, forge.EventSuccessfulReflection, forge.EventSuccessfulReflectionMsg()) + klog.V(4).Infof("Handle of local PersistentVolumeClaim %q (remote: %q) finished", nls.LocalRef(name), nls.RemoteRef(name)) + return nil +} + +// List lists all local-storage PersistentVolumeClaims in the local cluster. +func (nls *NamespacedLocalStoragePVCReflector) List() ([]interface{}, error) { + return virtualkubelet.List[virtualkubelet.Lister[*corev1.PersistentVolumeClaim], *corev1.PersistentVolumeClaim]( + nls.localPersistentVolumeClaims, + nls.remotePersistentVolumeClaims, + ) +} + +func (nls *NamespacedLocalStoragePVCReflector) deleteRemotePV(ctx context.Context, name string) error { + remotePV, err := nls.remotePersistentVolumes.Get(name) + if kerrors.IsNotFound(err) { + klog.V(4).Infof("Remote PersistentVolume %q not found, skipping deletion", name) + return nil + } + if err != nil { + return err + } + + if !isLocalStorageReflectedByUs(remotePV) { + klog.V(4).Infof("Skipping deletion of remote PersistentVolume %q since it is not managed by us (labels: %v)", name, remotePV.GetLabels()) + return nil + } + + if !isLocalStoragePV(remotePV) { + klog.Infof("Skipping deletion of remote PersistentVolume %q since it is not a local-storage volume)", name) + return nil + } + + klog.V(4).Infof("Deleting remote PersistentVolume %q", name) + return nls.remotePersistentVolumesClient.Delete(ctx, name, metav1.DeleteOptions{}) +} + +// isLocalStorageReflectedByUs returns whether the given object has been reflected by the local-storage PVC reflector. +func isLocalStorageReflectedByUs(obj metav1.Object) bool { + if !forge.IsReflected(obj) { + return false + } + + return obj.GetLabels()[forge.LiqoReflectedByLabelKey] == LocalStoragePVCReflectorName +} + +// isLocalStoragePV returns whether the given PV is backed by local storage (i.e. it has a Local volume or hostPath source). +func isLocalStoragePV(pv *corev1.PersistentVolume) bool { + return pv.Spec.Local != nil || pv.Spec.HostPath != nil +} + +// remotePVNameFromLocal returns the name to use for the reflected copy of the given local PV. +func remotePVNameFromLocal(localPV *corev1.PersistentVolume) string { + pvName := fmt.Sprintf("%s-%s", localPV.Name, localPV.UID[len(localPV.UID)-12:]) + if len(pvName) > 253 { + pvName = fmt.Sprintf("omnipv-%s", localPV.UID) + } + return pvName +} + +// forgeRemotePVC forges the apply patch for the reflected PersistentVolumeClaim. +func forgeRemotePVC(local *corev1.PersistentVolumeClaim, remoteNamespace, remotePVName string, + forgingOpts *forge.ForgingOpts) *corev1apply.PersistentVolumeClaimApplyConfiguration { + return corev1apply.PersistentVolumeClaim(local.Name, remoteNamespace). + WithLabels(forge.FilterNotReflected(local.GetLabels(), forgingOpts.LabelsNotReflected)). + WithLabels(forge.ReflectionLabels()). + WithLabels(labels.Set{forge.LiqoReflectedByLabelKey: LocalStoragePVCReflectorName}). + WithAnnotations(forge.FilterNotReflected(filterPVCAnnotations(local.GetAnnotations()), forgingOpts.AnnotationsNotReflected)). + WithSpec(forgeRemotePVCSpec(local, remotePVName)) +} + +func forgeRemotePVCSpec(local *corev1.PersistentVolumeClaim, remotePVName string) *corev1apply.PersistentVolumeClaimSpecApplyConfiguration { + res := corev1apply.PersistentVolumeClaimSpec(). + WithAccessModes(local.Spec.AccessModes...). + WithVolumeMode(func() corev1.PersistentVolumeMode { + if local.Spec.VolumeMode != nil { + return *local.Spec.VolumeMode + } + return corev1.PersistentVolumeFilesystem + }()). + WithResources(volumeResourceRequirements(local.Spec.Resources)). + WithVolumeName(remotePVName) + + if local.Spec.StorageClassName != nil && *local.Spec.StorageClassName != "" { + res.WithStorageClassName(*local.Spec.StorageClassName) + } + + return res +} + +// forgeRemotePV forges the apply patch for the reflected PersistentVolume. +func forgeRemotePV(localPV *corev1.PersistentVolume, remotePVC *corev1.PersistentVolumeClaim, + remoteNamespace string, forgingOpts *forge.ForgingOpts) *corev1apply.PersistentVolumeApplyConfiguration { + name := remotePVNameFromLocal(localPV) + return corev1apply.PersistentVolume(name). + WithLabels(forge.FilterNotReflected(localPV.GetLabels(), forgingOpts.LabelsNotReflected)). + WithLabels(forge.ReflectionLabels()). + WithLabels(labels.Set{forge.LiqoReflectedByLabelKey: LocalStoragePVCReflectorName}). + WithAnnotations(forge.FilterNotReflected(filterPVAnnotations(localPV.GetAnnotations()), forgingOpts.AnnotationsNotReflected)). + WithSpec(forgeRemotePVSpec(localPV, remotePVC, remoteNamespace)) +} + +func forgeRemotePVSpec(localPV *corev1.PersistentVolume, remotePVC *corev1.PersistentVolumeClaim, + remoteNamespace string) *corev1apply.PersistentVolumeSpecApplyConfiguration { + res := corev1apply.PersistentVolumeSpec(). + WithCapacity(localPV.Spec.Capacity). + WithAccessModes(localPV.Spec.AccessModes...). + WithVolumeMode(func() corev1.PersistentVolumeMode { + if localPV.Spec.VolumeMode != nil { + return *localPV.Spec.VolumeMode + } + return corev1.PersistentVolumeFilesystem + }()). + WithStorageClassName(localPV.Spec.StorageClassName). + WithClaimRef(corev1apply.ObjectReference(). + WithAPIVersion("v1"). + WithKind("PersistentVolumeClaim"). + WithNamespace(remoteNamespace). + WithName(remotePVC.Name). + WithUID(remotePVC.UID)). + WithPersistentVolumeReclaimPolicy(localPV.Spec.PersistentVolumeReclaimPolicy). + WithMountOptions(localPV.Spec.MountOptions...). + WithNodeAffinity(sanitizeNodeAffinity(localPV.Spec.NodeAffinity)) + + applyPVSource(res, localPV.Spec.PersistentVolumeSource.DeepCopy()) + + return res +} + +func applyPVSource(res *corev1apply.PersistentVolumeSpecApplyConfiguration, src *corev1.PersistentVolumeSource) { + switch { + case src.Local != nil: + local := corev1apply.LocalVolumeSource().WithPath(src.Local.Path) + if src.Local.FSType != nil { + local.WithFSType(*src.Local.FSType) + } + res.WithLocal(local) + case src.HostPath != nil: + hostPath := corev1apply.HostPathVolumeSource().WithPath(src.HostPath.Path) + if src.HostPath.Type != nil { + hostPath.WithType(*src.HostPath.Type) + } + res.WithHostPath(hostPath) + } + // For local-storage PVCs we expect only Local or HostPath sources. + // Other volume sources are intentionally ignored, as they are not supported + // by this reflector. +} + +func sanitizeNodeAffinity(affinity *corev1.VolumeNodeAffinity) *corev1apply.VolumeNodeAffinityApplyConfiguration { + if affinity == nil || affinity.Required == nil { + return dummyNodeAffinity() + } + + var terms []*corev1apply.NodeSelectorTermApplyConfiguration + for i := range affinity.Required.NodeSelectorTerms { + term := &affinity.Required.NodeSelectorTerms[i] + matchExpressions := filterNodeSelectorRequirements(term.MatchExpressions) + matchFields := filterNodeSelectorRequirements(term.MatchFields) + if len(matchExpressions) == 0 && len(matchFields) == 0 { + continue + } + terms = append(terms, corev1apply.NodeSelectorTerm(). + WithMatchExpressions(matchExpressions...). + WithMatchFields(matchFields...)) + } + + if len(terms) == 0 { + return dummyNodeAffinity() + } + + return corev1apply.VolumeNodeAffinity(). + WithRequired(corev1apply.NodeSelector().WithNodeSelectorTerms(terms...)) +} + +func filterNodeSelectorRequirements(reqs []corev1.NodeSelectorRequirement) []*corev1apply.NodeSelectorRequirementApplyConfiguration { + var res []*corev1apply.NodeSelectorRequirementApplyConfiguration + for i := range reqs { + req := &reqs[i] + if req.Key == consts.TypeLabel { + // Drop Liqo virtual-node type selectors; they are meaningless in the remote cluster. + continue + } + res = append(res, corev1apply.NodeSelectorRequirement(). + WithKey(req.Key). + WithOperator(req.Operator). + WithValues(req.Values...)) + } + return res +} + +func dummyNodeAffinity() *corev1apply.VolumeNodeAffinityApplyConfiguration { + return corev1apply.VolumeNodeAffinity(). + WithRequired(corev1apply.NodeSelector(). + WithNodeSelectorTerms(corev1apply.NodeSelectorTerm(). + WithMatchExpressions(corev1apply.NodeSelectorRequirement(). + WithKey("kubernetes.io/os"). + WithOperator(corev1.NodeSelectorOpExists)))) +} + +func volumeResourceRequirements(resources corev1.VolumeResourceRequirements) *corev1apply.VolumeResourceRequirementsApplyConfiguration { + return corev1apply.VolumeResourceRequirements(). + WithLimits(resources.Limits). + WithRequests(resources.Requests) +} + +func filterPVCAnnotations(annotations map[string]string) map[string]string { + return maps.Filter(annotations, maps.FilterBlacklist(pvcAnnotationsBlacklist...)) +} + +func filterPVAnnotations(annotations map[string]string) map[string]string { + return maps.Filter(annotations, maps.FilterBlacklist(pvAnnotationsBlacklist...)) +} + +const ( + annBindCompleted = "pv.kubernetes.io/bind-completed" + annBoundByController = "pv.kubernetes.io/bound-by-controller" + annMigratedTo = "pv.kubernetes.io/migrated-to" + annKubectlLastApplied = "kubectl.kubernetes.io/last-applied-configuration" + annStorageAlphaSelectedNode = "storage.alpha.kubernetes.io/selected-node" + annGAStorageProvisioner = "volume.kubernetes.io/storage-provisioner" +) + +var pvcAnnotationsBlacklist = []string{ + annBindCompleted, + annBoundByController, + annDynamicallyProvisioned, + annMigratedTo, + annStorageProvisioner, + annGAStorageProvisioner, + annSelectedNode, + annAlphaSelectedNode, + annStorageAlphaSelectedNode, + annKubectlLastApplied, +} + +var pvAnnotationsBlacklist = []string{ + annBindCompleted, + annBoundByController, + annDynamicallyProvisioned, + annMigratedTo, + annStorageProvisioner, + annGAStorageProvisioner, + annSelectedNode, + annAlphaSelectedNode, + annStorageAlphaSelectedNode, + annKubectlLastApplied, +} diff --git a/pkg/virtualKubelet/reflection/storage/localstoragepvc_test.go b/pkg/virtualKubelet/reflection/storage/localstoragepvc_test.go new file mode 100644 index 0000000000..e3b7885e2c --- /dev/null +++ b/pkg/virtualKubelet/reflection/storage/localstoragepvc_test.go @@ -0,0 +1,510 @@ +// Copyright 2019-2026 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/tools/record" + "k8s.io/utils/ptr" + + offloadingv1beta1 "github.com/liqotech/liqo/apis/offloading/v1beta1" + "github.com/liqotech/liqo/pkg/consts" + "github.com/liqotech/liqo/pkg/utils/testutil" + "github.com/liqotech/liqo/pkg/virtualKubelet/forge" + "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/options" +) + +const ( + localStoragePVCName = "local-storage-pvc" + localPVName = "local-storage-pv" +) + +// awaitLocalCacheEntries waits until the given local PVC (and, when non-empty, the PV) are visible +// in the shared informers' caches. WaitForCacheSync returns immediately if the informers were +// already started by a previous test, in which case a freshly created object may not be listed +// yet, and the subsequent Handle call would wrongly consider it as vanished. +func awaitLocalCacheEntries(pvcName, pvName string) { + Eventually(func() bool { + if _, err := factory.Core().V1().PersistentVolumeClaims().Lister(). + PersistentVolumeClaims(LocalNamespace).Get(pvcName); err != nil { + return false + } + if pvName == "" { + return true + } + _, err := factory.Core().V1().PersistentVolumes().Lister().Get(pvName) + return err == nil + }, 10*time.Second, 250*time.Millisecond).Should(BeTrue()) +} + +// createBoundLocalStoragePVC creates a bound local-storage PVC together with its backing PV, +// and returns the created PV and PVC objects. The PVC uses the standard local-storage class +// and is bound to the returned PV. +func createBoundLocalStoragePVC(pvcName, pvName string, annotations map[string]string) (*corev1.PersistentVolume, *corev1.PersistentVolumeClaim) { + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvName, + }, + Spec: corev1.PersistentVolumeSpec{ + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + Local: &corev1.LocalVolumeSource{Path: "/mnt/disks/vol1"}, + }, + StorageClassName: "local-storage", + NodeAffinity: &corev1.VolumeNodeAffinity{ + Required: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + {MatchExpressions: []corev1.NodeSelectorRequirement{ + {Key: "kubernetes.io/hostname", Operator: corev1.NodeSelectorOpIn, Values: []string{"real-node"}}, + }}, + }, + }, + }, + }, + } + + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: LocalNamespace, + Annotations: annotations, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: ptr.To("local-storage"), + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")}, + }, + VolumeName: pvName, + }, + } + + return pv, pvc +} + +var _ = Describe("local storage PVC reflector helpers", func() { + + Context("isLocalStoragePV", func() { + type isLocalStorageTestcase struct { + pv *corev1.PersistentVolume + expected OmegaMatcher + } + + DescribeTable("isLocalStoragePV table", + func(c isLocalStorageTestcase) { + Expect(isLocalStoragePV(c.pv)).To(c.expected) + }, + Entry("local volume source", isLocalStorageTestcase{ + pv: &corev1.PersistentVolume{ + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + Local: &corev1.LocalVolumeSource{Path: "/mnt/disks/vol1"}, + }, + }, + }, + expected: BeTrue(), + }), + Entry("host path volume source", isLocalStorageTestcase{ + pv: &corev1.PersistentVolume{ + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + HostPath: &corev1.HostPathVolumeSource{Path: "/tmp"}, + }, + }, + }, + expected: BeTrue(), + }), + Entry("nil volume source", isLocalStorageTestcase{ + pv: &corev1.PersistentVolume{ + Spec: corev1.PersistentVolumeSpec{}, + }, + expected: BeFalse(), + }), + ) + }) + + Context("sanitizeNodeAffinity", func() { + It("should return a dummy selector when affinity is nil", func() { + res := sanitizeNodeAffinity(nil) + Expect(res).ToNot(BeNil()) + Expect(res.Required).ToNot(BeNil()) + Expect(res.Required.NodeSelectorTerms).To(HaveLen(1)) + Expect(res.Required.NodeSelectorTerms[0].MatchExpressions).To(HaveLen(1)) + Expect(*res.Required.NodeSelectorTerms[0].MatchExpressions[0].Key).To(Equal("kubernetes.io/os")) + Expect(*res.Required.NodeSelectorTerms[0].MatchExpressions[0].Operator). + To(Equal(corev1.NodeSelectorOpExists)) + }) + + It("should strip virtual-node selectors and preserve real-node selectors", func() { + affinity := &corev1.VolumeNodeAffinity{ + Required: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + {Key: consts.TypeLabel, Operator: corev1.NodeSelectorOpIn, Values: []string{consts.TypeNode}}, + {Key: "kubernetes.io/hostname", Operator: corev1.NodeSelectorOpIn, Values: []string{"real-node"}}, + }, + }, + }, + }, + } + res := sanitizeNodeAffinity(affinity) + Expect(res.Required.NodeSelectorTerms).To(HaveLen(1)) + Expect(res.Required.NodeSelectorTerms[0].MatchExpressions).To(HaveLen(1)) + Expect(*res.Required.NodeSelectorTerms[0].MatchExpressions[0].Key).To(Equal("kubernetes.io/hostname")) + }) + + It("should add a dummy selector when all terms are stripped", func() { + affinity := &corev1.VolumeNodeAffinity{ + Required: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + {Key: consts.TypeLabel, Operator: corev1.NodeSelectorOpIn, Values: []string{consts.TypeNode}}, + }, + }, + }, + }, + } + res := sanitizeNodeAffinity(affinity) + Expect(res.Required.NodeSelectorTerms).To(HaveLen(1)) + Expect(*res.Required.NodeSelectorTerms[0].MatchExpressions[0].Key).To(Equal("kubernetes.io/os")) + }) + }) + + Context("filterPVCAnnotations", func() { + It("should strip cluster-specific annotations", func() { + annotations := map[string]string{ + "pv.kubernetes.io/bind-completed": "yes", + "volume.kubernetes.io/selected-node": "node-1", + "volume.kubernetes.io/storage-provisioner": "local-volume-provisioner", + "keep-this": "value", + } + res := filterPVCAnnotations(annotations) + Expect(res).ToNot(HaveKey("pv.kubernetes.io/bind-completed")) + Expect(res).ToNot(HaveKey("volume.kubernetes.io/selected-node")) + Expect(res).ToNot(HaveKey("volume.kubernetes.io/storage-provisioner")) + Expect(res).To(HaveKeyWithValue("keep-this", "value")) + }) + }) + + Context("remotePVNameFromLocal", func() { + It("should generate a deterministic name from the local PV UID", func() { + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "local-pv", + UID: "f16f07c4-ad34-45d4-b6ad-ef233e40b90a", + }, + } + Expect(remotePVNameFromLocal(pv)).To(Equal("local-pv-ef233e40b90a")) + }) + It("should generate a shorter name if pv name is longer than 253 characters", func() { + longName := strings.Repeat("a", 253) + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: longName, + UID: "f16f07c4-ad34-45d4-b6ad-ef233e40b90a", + }, + } + Expect(remotePVNameFromLocal(pv)).To(Equal("omnipv-f16f07c4-ad34-45d4-b6ad-ef233e40b90a")) + }) + }) +}) + +var _ = Describe("local storage PVC reflector integration", func() { + var lsReflector *NamespacedLocalStoragePVCReflector + + BeforeEach(func() { + lsReflectorBuilder := NewNamespacedLocalStoragePVCReflector() + opts := options.NewNamespaced(). + WithLocal(LocalNamespace, k8sClient, factory). + WithRemote(RemoteNamespace, k8sClient, factory). + WithHandlerFactory(FakeEventHandler). + WithEventBroadcaster(record.NewBroadcaster()). + WithForgingOpts(testutil.FakeForgingOpts()). + WithReflectionType(offloadingv1beta1.DenyList) + + lsReflector = lsReflectorBuilder(opts).(*NamespacedLocalStoragePVCReflector) + Expect(lsReflector).ToNot(BeNil()) + }) + + It("should reflect a bound local-storage PVC and its PV", func() { + pv, pvc := createBoundLocalStoragePVC(localStoragePVCName, localPVName, + map[string]string{ + "volume.kubernetes.io/selected-node": "local-node", + "volume.kubernetes.io/storage-provisioner": "local-volume-provisioner", + testutil.FakeNotReflectedAnnotKey: "true", + }) + pv.Annotations = map[string]string{ + "pv.kubernetes.io/provisioned-by": "local-volume-provisioner", + } + + pv.Spec.NodeAffinity = &corev1.VolumeNodeAffinity{ + Required: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + {Key: consts.TypeLabel, Operator: corev1.NodeSelectorOpIn, Values: []string{consts.TypeNode}}, + }, + }, + }, + }, + } + + pv, err := k8sClient.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + _, err = k8sClient.CoreV1().PersistentVolumeClaims(LocalNamespace).Create(ctx, pvc, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + factory.Start(ctx.Done()) + factory.WaitForCacheSync(ctx.Done()) + + awaitLocalCacheEntries(localStoragePVCName, pv.Name) + + Expect(lsReflector.Handle(ctx, localStoragePVCName)).To(Succeed()) + + expectedRemotePVName := remotePVNameFromLocal(pv) + + // Verify the remote PVC. + remotePVC, err := k8sClient.CoreV1().PersistentVolumeClaims(RemoteNamespace). + Get(ctx, localStoragePVCName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(remotePVC.Spec.StorageClassName).To(HaveValue(Equal("local-storage"))) + Expect(remotePVC.Spec.VolumeName).To(Equal(expectedRemotePVName)) + Expect(remotePVC.Labels).To(HaveKey(forge.LiqoOriginClusterIDKey)) + Expect(remotePVC.Labels).To(HaveKeyWithValue(forge.LiqoReflectedByLabelKey, LocalStoragePVCReflectorName)) + Expect(remotePVC.Annotations).ToNot(HaveKey("volume.kubernetes.io/selected-node")) + Expect(remotePVC.Annotations).ToNot(HaveKey("volume.kubernetes.io/storage-provisioner")) + + // Verify the remote PV. + remotePV, err := k8sClient.CoreV1().PersistentVolumes().Get(ctx, expectedRemotePVName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(remotePV.Spec.StorageClassName).To(Equal("local-storage")) + Expect(remotePV.Spec.Local.Path).To(Equal("/mnt/disks/vol1")) + Expect(remotePV.Spec.ClaimRef.Namespace).To(Equal(RemoteNamespace)) + Expect(remotePV.Spec.ClaimRef.Name).To(Equal(localStoragePVCName)) + Expect(remotePV.Labels).To(HaveKeyWithValue(forge.LiqoReflectedByLabelKey, LocalStoragePVCReflectorName)) + Expect(remotePV.Annotations).ToNot(HaveKey("pv.kubernetes.io/provisioned-by")) + Expect(remotePV.Spec.NodeAffinity.Required.NodeSelectorTerms).To(HaveLen(1)) + Expect(remotePV.Spec.NodeAffinity.Required.NodeSelectorTerms[0].MatchExpressions).To(HaveLen(1)) + Expect(remotePV.Spec.NodeAffinity.Required.NodeSelectorTerms[0].MatchExpressions[0].Key).To(Equal("kubernetes.io/os")) + Expect(remotePV.Spec.NodeAffinity.Required.NodeSelectorTerms[0].MatchExpressions[0].Operator). + To(Equal(corev1.NodeSelectorOpExists)) + }) + + It("should not reflect a local-storage PVC that is not bound", func() { + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "unbound-local-storage-pvc", + Namespace: LocalNamespace, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: ptr.To("local-storage"), + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + } + + _, err := k8sClient.CoreV1().PersistentVolumeClaims(LocalNamespace).Create(ctx, pvc, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + factory.Start(ctx.Done()) + factory.WaitForCacheSync(ctx.Done()) + + Expect(lsReflector.Handle(ctx, "unbound-local-storage-pvc")).To(Succeed()) + + _, err = k8sClient.CoreV1().PersistentVolumeClaims(RemoteNamespace). + Get(ctx, "unbound-local-storage-pvc", metav1.GetOptions{}) + Expect(err).To(HaveOccurred()) + }) + + It("should delete the remote PVC and PV when the local PVC is deleted", func() { + deleteTestPVName := "delete-test-local-pv" + deleteTestPVCName := "delete-test-local-storage-pvc" + + pv, pvc := createBoundLocalStoragePVC(deleteTestPVCName, deleteTestPVName, + map[string]string{consts.AllowReflectionAnnotationKey: "true"}) + + pv, err := k8sClient.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + _, err = k8sClient.CoreV1().PersistentVolumeClaims(LocalNamespace).Create(ctx, pvc, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + factory.Start(ctx.Done()) + factory.WaitForCacheSync(ctx.Done()) + + awaitLocalCacheEntries(deleteTestPVCName, deleteTestPVName) + + Expect(lsReflector.Handle(ctx, deleteTestPVCName)).To(Succeed()) + + remotePVName := remotePVNameFromLocal(pv) + _, err = k8sClient.CoreV1().PersistentVolumes().Get(ctx, remotePVName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + + // Delete the local PVC via the API. In envtest, the PVC protection + // finalizer is not automatically removed, so we clear it to let the + // deletion complete. Then create a fresh reflector so that its informer + // lists the current state (PVC missing) and triggers cleanup of the + // remote objects. + Expect(k8sClient.CoreV1().PersistentVolumeClaims(LocalNamespace). + Delete(ctx, deleteTestPVCName, metav1.DeleteOptions{})).To(Succeed()) + deletedPVC, err := k8sClient.CoreV1().PersistentVolumeClaims(LocalNamespace). + Get(ctx, deleteTestPVCName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + deletedPVC.Finalizers = nil + _, err = k8sClient.CoreV1().PersistentVolumeClaims(LocalNamespace). + Update(ctx, deletedPVC, metav1.UpdateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + freshFactory := informers.NewSharedInformerFactory(k8sClient, 10*time.Hour) + freshOpts := options.NewNamespaced(). + WithLocal(LocalNamespace, k8sClient, freshFactory). + WithRemote(RemoteNamespace, k8sClient, freshFactory). + WithHandlerFactory(FakeEventHandler). + WithEventBroadcaster(record.NewBroadcaster()). + WithForgingOpts(testutil.FakeForgingOpts()). + WithReflectionType(offloadingv1beta1.DenyList) + freshReflector := NewNamespacedLocalStoragePVCReflector()(freshOpts).(*NamespacedLocalStoragePVCReflector) + freshFactory.Start(ctx.Done()) + freshFactory.WaitForCacheSync(ctx.Done()) + + Expect(freshReflector.Handle(ctx, deleteTestPVCName)).To(Succeed()) + + // In envtest, protection finalizers can delay deletion. If the objects + // are still present with a deletion timestamp, clear their finalizers to + // let the deletion complete, then assert they are gone. + remotePVC, err := k8sClient.CoreV1().PersistentVolumeClaims(RemoteNamespace). + Get(ctx, deleteTestPVCName, metav1.GetOptions{}) + if err == nil && remotePVC.DeletionTimestamp != nil { + remotePVC.Finalizers = nil + _, _ = k8sClient.CoreV1().PersistentVolumeClaims(RemoteNamespace). + Update(ctx, remotePVC, metav1.UpdateOptions{}) + } + remotePV, err := k8sClient.CoreV1().PersistentVolumes().Get(ctx, remotePVName, metav1.GetOptions{}) + if err == nil && remotePV.DeletionTimestamp != nil { + remotePV.Finalizers = nil + _, _ = k8sClient.CoreV1().PersistentVolumes().Update(ctx, remotePV, metav1.UpdateOptions{}) + } + + _, err = k8sClient.CoreV1().PersistentVolumeClaims(RemoteNamespace). + Get(ctx, deleteTestPVCName, metav1.GetOptions{}) + Expect(err).To(HaveOccurred()) + _, err = k8sClient.CoreV1().PersistentVolumes().Get(ctx, remotePVName, metav1.GetOptions{}) + Expect(err).To(HaveOccurred()) + }) + + Context("when the local PVC is marked with the skip annotation", func() { + It("should not reflect a skipped local-storage PVC", func() { + pv, pvc := createBoundLocalStoragePVC("skipped-local-storage-pvc", "skipped-local-storage-pv", + map[string]string{consts.SkipReflectionAnnotationKey: "true"}) + + pv, err := k8sClient.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + pvc, err = k8sClient.CoreV1().PersistentVolumeClaims(LocalNamespace).Create(ctx, pvc, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + factory.Start(ctx.Done()) + factory.WaitForCacheSync(ctx.Done()) + + Expect(lsReflector.Handle(ctx, pvc.Name)).To(Succeed()) + + _, err = k8sClient.CoreV1().PersistentVolumeClaims(RemoteNamespace). + Get(ctx, pvc.Name, metav1.GetOptions{}) + Expect(err).To(HaveOccurred()) + _, err = k8sClient.CoreV1().PersistentVolumes().Get(ctx, remotePVNameFromLocal(pv), metav1.GetOptions{}) + Expect(err).To(HaveOccurred()) + }) + }) + + Context("when the reflection type is AllowList", func() { + BeforeEach(func() { + allowOpts := options.NewNamespaced(). + WithLocal(LocalNamespace, k8sClient, factory). + WithRemote(RemoteNamespace, k8sClient, factory). + WithHandlerFactory(FakeEventHandler). + WithEventBroadcaster(record.NewBroadcaster()). + WithForgingOpts(testutil.FakeForgingOpts()). + WithReflectionType(offloadingv1beta1.AllowList) + + lsReflector = NewNamespacedLocalStoragePVCReflector()(allowOpts).(*NamespacedLocalStoragePVCReflector) + Expect(lsReflector).ToNot(BeNil()) + }) + + It("should not reflect a local-storage PVC without the allow annotation", func() { + pv, pvc := createBoundLocalStoragePVC("not-allowed-local-storage-pvc", "not-allowed-local-storage-pv", nil) + + pv, err := k8sClient.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + pvc, err = k8sClient.CoreV1().PersistentVolumeClaims(LocalNamespace).Create(ctx, pvc, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + factory.Start(ctx.Done()) + factory.WaitForCacheSync(ctx.Done()) + + Expect(lsReflector.Handle(ctx, pvc.Name)).To(Succeed()) + + _, err = k8sClient.CoreV1().PersistentVolumeClaims(RemoteNamespace). + Get(ctx, pvc.Name, metav1.GetOptions{}) + Expect(err).To(HaveOccurred()) + _, err = k8sClient.CoreV1().PersistentVolumes().Get(ctx, remotePVNameFromLocal(pv), metav1.GetOptions{}) + Expect(err).To(HaveOccurred()) + }) + + It("should reflect a local-storage PVC with the allow annotation", func() { + pv, pvc := createBoundLocalStoragePVC("allowed-local-storage-pvc", "allowed-local-storage-pv", + map[string]string{consts.AllowReflectionAnnotationKey: "true"}) + + pv, err := k8sClient.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + pvc, err = k8sClient.CoreV1().PersistentVolumeClaims(LocalNamespace).Create(ctx, pvc, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + factory.Start(ctx.Done()) + factory.WaitForCacheSync(ctx.Done()) + + awaitLocalCacheEntries(pvc.Name, pv.Name) + + Expect(lsReflector.Handle(ctx, pvc.Name)).To(Succeed()) + + expectedRemotePVName := remotePVNameFromLocal(pv) + + remotePVC, err := k8sClient.CoreV1().PersistentVolumeClaims(RemoteNamespace). + Get(ctx, pvc.Name, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(remotePVC.Spec.VolumeName).To(Equal(expectedRemotePVName)) + Expect(remotePVC.Labels).To(HaveKeyWithValue(forge.LiqoReflectedByLabelKey, LocalStoragePVCReflectorName)) + + remotePV, err := k8sClient.CoreV1().PersistentVolumes().Get(ctx, expectedRemotePVName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(remotePV.Spec.Local.Path).To(Equal("/mnt/disks/vol1")) + Expect(remotePV.Labels).To(HaveKeyWithValue(forge.LiqoReflectedByLabelKey, LocalStoragePVCReflectorName)) + }) + }) +}) diff --git a/pkg/virtualKubelet/reflection/storage/persistentvolumeclaim.go b/pkg/virtualKubelet/reflection/storage/persistentvolumeclaim.go index e30c1de16a..ec4937f5e6 100644 --- a/pkg/virtualKubelet/reflection/storage/persistentvolumeclaim.go +++ b/pkg/virtualKubelet/reflection/storage/persistentvolumeclaim.go @@ -29,6 +29,7 @@ import ( "k8s.io/utils/trace" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/sig-storage-lib-external-provisioner/v7/controller" + "sigs.k8s.io/sig-storage-lib-external-provisioner/v7/util" offloadingv1beta1 "github.com/liqotech/liqo/apis/offloading/v1beta1" "github.com/liqotech/liqo/pkg/consts" @@ -118,6 +119,7 @@ func (npvcr *NamespacedPersistentVolumeClaimReflector) Handle(ctx context.Contex klog.V(4).Infof("Handling reflection of local PersistentVolumeClaim %q (remote: %q)", npvcr.LocalRef(name), npvcr.RemoteRef(name)) local, lerr := npvcr.localPersistentVolumeClaims.Get(name) utilruntime.Must(client.IgnoreNotFound(lerr)) + remote, rerr := npvcr.remotePersistentVolumeClaims.Get(name) utilruntime.Must(client.IgnoreNotFound(rerr)) tracer.Step("Retrieved the local and remote objects") @@ -176,12 +178,19 @@ func (npvcr *NamespacedPersistentVolumeClaimReflector) Handle(ctx context.Contex // In addition, we use the provisionFunc to ensure the provisioning of the remote PVC. state, err := npvcr.provisionClaimOperation(ctx, local, func(_ context.Context, options controller.ProvisionOptions) (*corev1.PersistentVolume, controller.ProvisioningState, error) { - if clusterID, found := utils.GetNodeClusterID(options.SelectedNode); !found || clusterID != string(forge.RemoteCluster) { - return nil, controller.ProvisioningFinished, &controller.IgnoredError{Reason: "this provisioner is not provisioning storage on that node"} + // Check whether the pvc should be provisioned on all the edges + shouldProvisionOnAllEdges := local.Annotations != nil && + local.Annotations[consts.ProvisionPVCOnAllEdgesAnnotationKey] == consts.ProvisionPVCOnAllEdgesAnnotationValue + + if !shouldProvisionOnAllEdges { + if clusterID, found := utils.GetNodeClusterID(options.SelectedNode); !found || clusterID != string(forge.RemoteCluster) { + return nil, controller.ProvisioningFinished, &controller.IgnoredError{Reason: "this provisioner is not provisioning storage on that node"} + } } pv, state, err := liqostorageprovisioner.ProvisionRemotePVC(ctx, - options, npvcr.RemoteNamespace(), npvcr.remoteRealStorageClassName, + options, npvcr.RemoteNamespace(), + util.GetPersistentVolumeClaimClass(local), npvcr.remoteRealStorageClassName, npvcr.remotePersistentVolumeClaims, npvcr.remotePersistentVolumesClaimsClient, npvcr.ForgingOpts) if err == nil && state == controller.ProvisioningFinished { diff --git a/pkg/virtualKubelet/reflection/storage/persistentvolumeclaim_test.go b/pkg/virtualKubelet/reflection/storage/persistentvolumeclaim_test.go index 2919406f84..c650549ab0 100644 --- a/pkg/virtualKubelet/reflection/storage/persistentvolumeclaim_test.go +++ b/pkg/virtualKubelet/reflection/storage/persistentvolumeclaim_test.go @@ -100,6 +100,12 @@ var _ = Describe("reflector methods", func() { It("should create the local PV", func() { Expect(err).ToNot(HaveOccurred()) + // Retrieve the PVC directly: the shared localPvc variable is only populated by the + // sibling spec above, which is not guaranteed to have run first when the spec order + // is randomized. + localPvc, err := k8sClient.CoreV1().PersistentVolumeClaims(LocalNamespace).Get(ctx, remotePvcName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + localPv, err := k8sClient.CoreV1().PersistentVolumes().Get(ctx, localPvc.Spec.VolumeName, metav1.GetOptions{}) Expect(err).ToNot(HaveOccurred()) Expect(localPv).ToNot(BeNil()) diff --git a/pkg/virtualKubelet/reflection/storage/storage_suite_test.go b/pkg/virtualKubelet/reflection/storage/storage_suite_test.go index 52183580c1..390fd07877 100644 --- a/pkg/virtualKubelet/reflection/storage/storage_suite_test.go +++ b/pkg/virtualKubelet/reflection/storage/storage_suite_test.go @@ -46,8 +46,9 @@ const ( LocalNamespace = "local-namespace" RemoteNamespace = "remote-namespace" - LocalClusterID = "local-cluster-id" - RemoteClusterID = "remote-cluster-id" + LocalClusterID = "local-cluster-id" + RemoteClusterID = "remote-cluster-id" + EdgeLocationName = "fast-edge" VirtualStorageClassName = "liqo" RealRemoteStorageClassName = "other-class" @@ -142,7 +143,8 @@ var _ = BeforeEach(func() { ObjectMeta: metav1.ObjectMeta{ Name: VirtualNodeName, Labels: map[string]string{ - consts.RemoteClusterID: RemoteClusterID, + consts.RemoteClusterID: RemoteClusterID, + consts.EdgeLocationName: EdgeLocationName, }, }, } diff --git a/pkg/virtualKubelet/reflection/workload/podns.go b/pkg/virtualKubelet/reflection/workload/podns.go index d1ee952764..eb7f3734b7 100644 --- a/pkg/virtualKubelet/reflection/workload/podns.go +++ b/pkg/virtualKubelet/reflection/workload/podns.go @@ -337,12 +337,21 @@ func (npr *NamespacedPodReflector) ForgeShadowPod(ctx context.Context, local *co return ips } + // If the pod has ResourceClaims from templates, wait until the local kubelet has + // resolved all of them into concrete ResourceClaim names (written to Status.ResourceClaimStatuses). + // Until then, requeue rather than reflecting an incomplete spec to the remote. + if err := forge.CheckDRAClaimStatusesReady(local); err != nil { + return nil, fmt.Errorf("checking DRA claim statuses: %w", err) + } + var mutators []forge.RemotePodSpecMutator mutators = append(mutators, forge.APIServerSupportMutator(npr.config.APIServerSupport, local.Annotations, pod.ServiceAccountName(local), saSecretRetriever, ipsGetter, npr.config.HomeAPIServerHost, npr.config.HomeAPIServerPort), - forge.ServiceAccountMutator(npr.config.APIServerSupport, local.Annotations)) + forge.ServiceAccountMutator(npr.config.APIServerSupport, local.Annotations), + forge.DRAClaimRewriteMutator(local.Status.ResourceClaimStatuses), + ) if forgingOpts != nil { mutators = append(mutators, @@ -431,7 +440,10 @@ func (npr *NamespacedPodReflector) HandleStatus(ctx context.Context, local, remo info.RemoteUID = remote.GetUID() } - mutators := []forge.RemotePodStatusMutator{} + mutators := []forge.RemotePodStatusMutator{ + // ResourceClaimStatuses are managed locally never overwrite them with the remote pod's view. + forge.PreserveResourceClaimStatusesMutator(local.Status.ResourceClaimStatuses), + } if npr.config.DisableIPReflection { // If the IP reflection is disabled, we need to not reflect the IP address of the remote pod. mutators = append(mutators, forge.OpaqueIPTranslationMutator()) diff --git a/pkg/virtualKubelet/roles/local/role.go b/pkg/virtualKubelet/roles/local/role.go index 0b8143076d..6cadc3ff1a 100644 --- a/pkg/virtualKubelet/roles/local/role.go +++ b/pkg/virtualKubelet/roles/local/role.go @@ -42,3 +42,7 @@ package local // Additional permissions necessary for the networking module // +kubebuilder:rbac:groups=ipam.liqo.io,resources=ips,verbs=get;list;watch // +kubebuilder:rbac:groups=networking.liqo.io,resources=configurations,verbs=get;list;watch + +// Permissions for DRA reflection of ResourceSlices, ResourceClaims and DeviceClasses. +// +kubebuilder:rbac:groups=resource.k8s.io,resources=resourceslices,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=resource.k8s.io,resources=resourceclaims;deviceclasses,verbs=get;list;watch diff --git a/pkg/virtualKubelet/roles/remote/role.go b/pkg/virtualKubelet/roles/remote/role.go index 6730d75884..040e4f7119 100644 --- a/pkg/virtualKubelet/roles/remote/role.go +++ b/pkg/virtualKubelet/roles/remote/role.go @@ -28,3 +28,6 @@ package remote // +kubebuilder:rbac:groups=offloading.liqo.io,resources=shadowpods,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=offloading.liqo.io,resources=shadowendpointslices,verbs=get;list;watch;create;update;patch;delete + +// Permissions for DRA reflection of ResourceSlices, ResourceClaims and DeviceClasses. +// +kubebuilder:rbac:groups=resource.k8s.io,resources=resourceclaims,verbs=get;list;watch;create;update;patch;delete diff --git a/pkg/virtualKubelet/roles/remoteclusterwide/role.go b/pkg/virtualKubelet/roles/remoteclusterwide/role.go index f14773be53..5cc38e47d2 100644 --- a/pkg/virtualKubelet/roles/remoteclusterwide/role.go +++ b/pkg/virtualKubelet/roles/remoteclusterwide/role.go @@ -16,3 +16,9 @@ package remoteclusterwide // +kubebuilder:rbac:groups=metrics.liqo.io,resources=scrape/metrics,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch;create;update;patch;delete + +// Permissions for DRA reflection of ResourceSlices. +// +kubebuilder:rbac:groups=resource.k8s.io,resources=resourceslices,verbs=get;list;watch +// +kubebuilder:rbac:groups=resource.k8s.io,resources=deviceclasses,verbs=get;list;watch;create diff --git a/pkg/vkMachinery/forge/forge.go b/pkg/vkMachinery/forge/forge.go index b6c5dfc545..39263d9518 100644 --- a/pkg/vkMachinery/forge/forge.go +++ b/pkg/vkMachinery/forge/forge.go @@ -255,6 +255,8 @@ func forgeVKPodSpec(vkNamespace string, homeCluster liqov1beta1.ClusterID, liqoN ServiceAccountName: VirtualKubeletServiceAccountName(virtualNode.Name), ImagePullSecrets: opts.Spec.ImagePullSecrets, Tolerations: opts.Spec.Tolerations, + Affinity: opts.Spec.Affinity, + NodeSelector: opts.Spec.NodeSelector, } } diff --git a/test/e2e/cruise/network/network_test.go b/test/e2e/cruise/network/network_test.go index 1f10e13d73..d383e911b1 100644 --- a/test/e2e/cruise/network/network_test.go +++ b/test/e2e/cruise/network/network_test.go @@ -35,7 +35,6 @@ import ( liqov1beta1 "github.com/liqotech/liqo/apis/core/v1beta1" networkingv1beta1 "github.com/liqotech/liqo/apis/networking/v1beta1" "github.com/liqotech/liqo/pkg/gateway" - "github.com/liqotech/liqo/pkg/gateway/concurrent" networkflags "github.com/liqotech/liqo/pkg/liqoctl/test/network/flags" "github.com/liqotech/liqo/pkg/liqoctl/test/network/setup" "github.com/liqotech/liqo/test/e2e/testconsts" @@ -319,7 +318,6 @@ func RestartPods(cl client.Client) { cl.List(ctx, podList, &client.ListOptions{ LabelSelector: labels.SelectorFromSet(labels.Set{ gateway.GatewayComponentKey: gateway.GatewayComponentGateway, - concurrent.ActiveGatewayKey: concurrent.ActiveGatewayValue, }), }), ).To(Succeed()) @@ -381,7 +379,6 @@ func checkUniqueActiveGatewayPod(cl client.Client, numActiveGateway int) error { if err := cl.List(ctx, podList, &client.ListOptions{ LabelSelector: labels.SelectorFromSet(labels.Set{ gateway.GatewayComponentKey: gateway.GatewayComponentGateway, - concurrent.ActiveGatewayKey: concurrent.ActiveGatewayValue, }), }); err != nil { return fmt.Errorf("unable to list active gateway pods: %w", err) diff --git a/test/e2e/pipeline/installer/liqo/peer.sh b/test/e2e/pipeline/installer/liqo/peer.sh index be62996ed2..0a3c963dca 100755 --- a/test/e2e/pipeline/installer/liqo/peer.sh +++ b/test/e2e/pipeline/installer/liqo/peer.sh @@ -23,8 +23,8 @@ set -o pipefail # Fail if one of the piped commands fails set_certificate_renewal_policy() { local POLICY_MANIFEST POLICY_MANIFEST=$(cat <<'EOF' -apiVersion: kyverno.io/v1 -kind: ClusterPolicy +apiVersion: policies.kyverno.io/v1 +kind: MutatingPolicy metadata: name: patch-csr-expiration annotations: @@ -36,29 +36,39 @@ metadata: This policy patches CertificateSigningRequest resources with names starting with 'liqo-identity' to set the expiration duration to 600 seconds. spec: - rules: - - name: set-csr-expiration - match: - any: - - resources: - kinds: - - CertificateSigningRequest - names: - - "liqo-identity*" - mutate: - patchStrategicMerge: - spec: - expirationSeconds: 600 + matchConstraints: + resourceRules: + - apiGroups: + - certificates.k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - certificatesigningrequests + matchConditions: + - name: match-liqo-identity + expression: "object.metadata.name.startsWith('liqo-identity')" + mutations: + - patchType: ApplyConfiguration + applyConfiguration: + expression: | + Object{ + spec: Object.spec{ + expirationSeconds: 600 + } + } EOF ) for i in $(seq 1 "${CLUSTER_NUMBER}") do export KUBECONFIG="${TMPDIR}/kubeconfigs/liqo_kubeconf_${i}" - echo "Applying Kyverno ClusterPolicy on cluster ${i} to set the CSR expiration time to 600 seconds" + echo "Applying Kyverno MutatingPolicy on cluster ${i} to set the CSR expiration time to 600 seconds" echo "${POLICY_MANIFEST}" | "${KUBECTL}" apply -f - - echo "Waiting for Kyverno ClusterPolicy to become ready on cluster ${i}" - "${KUBECTL}" wait --for=condition=Ready clusterpolicy/patch-csr-expiration --timeout=120s + echo "Waiting for Kyverno MutatingPolicy to become ready on cluster ${i}" + "${KUBECTL}" wait --for=condition=Ready mutatingpolicy/patch-csr-expiration --timeout=120s done }