diff --git a/.github/scripts/gha-e2e.sh b/.github/scripts/gha-e2e.sh index 2c7cf04464e..d7fb8421fe5 100755 --- a/.github/scripts/gha-e2e.sh +++ b/.github/scripts/gha-e2e.sh @@ -104,9 +104,14 @@ function curvine_e2e() { set -e bash test/gha-e2e/curvine/test.sh } +function cache_selector_e2e() { + set -e + bash test/gha-e2e/cacheruntime-selector/test.sh +} check_control_plane_status alluxio_e2e jindo_e2e juicefs_e2e curvine_e2e +cache_selector_e2e diff --git a/api/v1alpha1/cacheruntime_types.go b/api/v1alpha1/cacheruntime_types.go index 51850a8a0f0..ce8f3f318c9 100644 --- a/api/v1alpha1/cacheruntime_types.go +++ b/api/v1alpha1/cacheruntime_types.go @@ -179,6 +179,7 @@ type CacheRuntimeSpec struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status +// +kubebuilder:subresource:scale:specpath=.spec.worker.replicas,statuspath=.status.worker.currentReplicas,selectorpath=.status.selector // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",priority=0 // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",priority=0 // +kubebuilder:resource:scope=Namespaced diff --git a/config/crd/bases/data.fluid.io_cacheruntimes.yaml b/config/crd/bases/data.fluid.io_cacheruntimes.yaml index 90bdd2a26d4..8f88108b63a 100644 --- a/config/crd/bases/data.fluid.io_cacheruntimes.yaml +++ b/config/crd/bases/data.fluid.io_cacheruntimes.yaml @@ -1656,4 +1656,8 @@ spec: served: true storage: true subresources: + scale: + labelSelectorPath: .status.selector + specReplicasPath: .spec.worker.replicas + statusReplicasPath: .status.worker.currentReplicas status: {} diff --git a/pkg/ddc/cache/engine/master.go b/pkg/ddc/cache/engine/master.go index ce82166d3c9..af0ee75bf5f 100644 --- a/pkg/ddc/cache/engine/master.go +++ b/pkg/ddc/cache/engine/master.go @@ -84,8 +84,7 @@ func (e *CacheEngine) setupMasterInternal(masterValue *common.CacheRuntimeCompon runtimeToUpdate := runtime.DeepCopy() runtimeToUpdate.Status.Master = masterStatus - // TODO(cache runtime): figure out how to use this selector - // runtimeToUpdate.Status.Selector = e.getWorkerSelectors() + runtimeToUpdate.Status.Selector = e.getWorkerSelectors() if len(runtimeToUpdate.Status.Conditions) == 0 { runtimeToUpdate.Status.Conditions = []datav1alpha1.RuntimeCondition{} diff --git a/pkg/ddc/cache/engine/status.go b/pkg/ddc/cache/engine/status.go index 5d3d744f2da..fb560ff85ff 100644 --- a/pkg/ddc/cache/engine/status.go +++ b/pkg/ddc/cache/engine/status.go @@ -149,7 +149,7 @@ func (e *CacheEngine) CheckAndUpdateRuntimeStatus(value *common.CacheRuntimeStat runtimeToUpdate.Status.SetupDuration = utils.CalculateDuration(runtimeToUpdate.CreationTimestamp.Time, time.Now()) } - // TODO(cache runtime): set the CacheRuntime Status left fields: Selector + runtimeToUpdate.Status.Selector = e.getWorkerSelectors() runtimeToUpdate.Status.ValueFile = common.GetCacheRuntimeConfigConfigMapName(e.name) if !reflect.DeepEqual(runtime.Status, runtimeToUpdate.Status) { diff --git a/pkg/ddc/cache/engine/util.go b/pkg/ddc/cache/engine/util.go index 00dc52eb3ca..04cfd0ecb08 100644 --- a/pkg/ddc/cache/engine/util.go +++ b/pkg/ddc/cache/engine/util.go @@ -23,6 +23,7 @@ import ( "github.com/fluid-cloudnative/fluid/pkg/common" "github.com/fluid-cloudnative/fluid/pkg/utils" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/validation" ) @@ -129,3 +130,15 @@ func GetEmptyDirTieredStoreMountPath(levelIndex int) string { func getTieredStoreMountPath(levelIndex int, pathIndex int, mediumType string) string { return fmt.Sprintf("/etc/fluid/mount/tiered-store/level-%d-index-%d-%s", levelIndex, pathIndex, mediumType) } + +// getWorkerSelectors returns the label selector string for CacheRuntime worker pods. +// This is used to populate Status.Selector so that HPA and other tooling can +// discover worker pods, matching the pattern used by other runtimes (JindoCache, +// JuiceFS, Vineyard, EFC). +func (e *CacheEngine) getWorkerSelectors() string { + workerName := common.GetCacheComponentName(e.name, common.ComponentTypeWorker) + return labels.SelectorFromSet(labels.Set{ + common.LabelCacheRuntimeName: e.name, + common.LabelCacheRuntimeComponentName: workerName, + }).String() +} diff --git a/pkg/ddc/cache/engine/util_test.go b/pkg/ddc/cache/engine/util_test.go index 440f0ce69f2..98dface6a06 100644 --- a/pkg/ddc/cache/engine/util_test.go +++ b/pkg/ddc/cache/engine/util_test.go @@ -19,6 +19,8 @@ package engine import ( "strings" + "github.com/fluid-cloudnative/fluid/pkg/common" + "github.com/fluid-cloudnative/fluid/pkg/utils/fake" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/util/validation" @@ -194,3 +196,51 @@ var _ = Describe("getSecretVolumeName Tests", Label("pkg.ddc.cache.engine.util_t }) }) }) + +var _ = Describe("getWorkerSelectors Tests", Label("pkg.ddc.cache.engine.util_test.go"), func() { + Describe("getWorkerSelectors", func() { + var engine *CacheEngine + + BeforeEach(func() { + engine = &CacheEngine{ + name: "test-runtime", + namespace: "default", + Log: fake.NullLogger(), + } + }) + + It("returns a non-empty selector string", func() { + selector := engine.getWorkerSelectors() + Expect(selector).NotTo(BeEmpty()) + }) + + It("selector contains the runtime name label", func() { + selector := engine.getWorkerSelectors() + Expect(selector).To(ContainSubstring(common.LabelCacheRuntimeName)) + Expect(selector).To(ContainSubstring("test-runtime")) + }) + + It("selector contains the worker component name label", func() { + workerName := common.GetCacheComponentName("test-runtime", common.ComponentTypeWorker) + selector := engine.getWorkerSelectors() + Expect(selector).To(ContainSubstring(common.LabelCacheRuntimeComponentName)) + Expect(selector).To(ContainSubstring(workerName)) + }) + + It("returns different selectors for different runtime names", func() { + engine1 := &CacheEngine{name: "runtime-a", namespace: "default", Log: fake.NullLogger()} + engine2 := &CacheEngine{name: "runtime-b", namespace: "default", Log: fake.NullLogger()} + Expect(engine1.getWorkerSelectors()).NotTo(Equal(engine2.getWorkerSelectors())) + }) + + It("returns the exact expected selector string", func() { + // Pin the selector string so any future change to label keys, ordering, + // or escaping is caught immediately. + engine := &CacheEngine{name: "test-runtime", namespace: "default", Log: fake.NullLogger()} + workerName := common.GetCacheComponentName("test-runtime", common.ComponentTypeWorker) + expected := common.LabelCacheRuntimeComponentName + "=" + workerName + + "," + common.LabelCacheRuntimeName + "=test-runtime" + Expect(engine.getWorkerSelectors()).To(Equal(expected)) + }) + }) +}) diff --git a/test/gha-e2e/cacheruntime-selector/cacheruntime.yaml b/test/gha-e2e/cacheruntime-selector/cacheruntime.yaml new file mode 100644 index 00000000000..e75735ddb33 --- /dev/null +++ b/test/gha-e2e/cacheruntime-selector/cacheruntime.yaml @@ -0,0 +1,24 @@ +apiVersion: data.fluid.io/v1alpha1 +kind: CacheRuntime +metadata: + name: selector-demo +spec: + runtimeClassName: selector-demo + master: + # 如何区分 master 和 journal 的配置,两个在一个进程中(前缀,交给Curvine自行处理) + options: # https://curvineio.github.io/zh-cn/docs/Deploy/Deploy-Curvine-Cluster/Distributed-Mode/conf#master%E9%85%8D%E7%BD%AE%E9%A1%B9 + key1: master-value1 + replicas: 1 + worker: + options: # https://curvineio.github.io/zh-cn/docs/Deploy/Deploy-Curvine-Cluster/Distributed-Mode/conf#worker%E9%85%8D%E7%BD%AE%E9%A1%B9 + key1: worker-value1 + replicas: 1 + tieredStore: + levels: #worker缓存配置 + - low: "0.5" + high: "0.8" + emptyDir: + quota: 1Gi + client: + options: + key1: value1 diff --git a/test/gha-e2e/cacheruntime-selector/cacheruntimeclass.yaml b/test/gha-e2e/cacheruntime-selector/cacheruntimeclass.yaml new file mode 100644 index 00000000000..64bde440391 --- /dev/null +++ b/test/gha-e2e/cacheruntime-selector/cacheruntimeclass.yaml @@ -0,0 +1,210 @@ +apiVersion: data.fluid.io/v1alpha1 +kind: CacheRuntimeClass +metadata: + name: selector-demo +fileSystemType: curvinefs +extraResources: + configMaps: + - name: curvine-config + data: + # TOML格式配置模板, 使用 hairyhenderson/gomplate 镜像要求的模板格式 + cluster.toml: | + # master configuration + [master] + meta_dir = "testing/meta" + + # masta ha raft configuration. + [journal] + journal_addrs = [ + {{ range $index := seq 0 (sub (ds "config").master.replicas 1) }} + {{- if $index }},{{ end }} + {id = {{ add $index 1 }}, hostname = "{{(ds "config").master.name}}-{{$index}}.{{(ds "config").master.service.name}}", port = 8996} + {{- end }} + ] + journal_dir = "testing/journal" + + # Worker configuration, translate Gi to GB + [worker] + dir_reserved = "0" + data_dir = [ + {{- range $index, $level := (ds "config").worker.tieredStoreLevels }} + {{- range $pathIndex, $mountPath := $level.mountPaths }} + {{- if or $index $pathIndex }},{{ end }} + "[{{$level.mediumType}}:{{ index $level.quotas $pathIndex | strings.ReplaceAll "i" "B"}}]{{ $mountPath }}" + {{- end }} + {{- end }} + ] +dataOperationSpecs: + - name: DataLoad + command: + - "/bin/bash" + - "-c" + args: + # Actually, the cache runtime image should use $(FLUID_RUNTIME_CONFIG_PATH) to generate the config file, and + # use $(FLUID_DATALOAD_DATA_PATH) to execute data load. + - | + # currently we have no customized image supporting dataload for curvine, so we write the curvine.toml with fixed journal address for test case. + echo -e '[journal]\njournal_addrs = [\n{id=1, hostname="selector-demo-master-0.svc-selector-demo-master"}\n]' > /etc/curvine.toml + + IFS=: read -ra paths <<< "$FLUID_DATALOAD_DATA_PATH" + for p in "${paths[@]}"; do + /app/curvine/bin/cv load "$p" --watch --conf /etc/curvine.toml || { + echo "Error: load $p failed." + exit 1 + } + done +topology: + master: + service: #需要为master创建Headless Service + headless: { } + dependencies: + extraResources: + # 使用 extraResources 时,需要定义其挂载路径 + configMaps: + - name: curvine-config + mountPath: "/templates" + executionEntries: + mountUFS: + command: + - bash + - "-c" + - "/etc/curvine/mount/mountUfs.sh" + timeout: 30 + template: + spec: + restartPolicy: Always + # 根据 runtime 生成的配置文件 + initContainers: + - name: init-curvine + image: hairyhenderson/gomplate:alpine + # 挂载共享卷到容器内路径 + volumeMounts: + - name: shared-config-volume + mountPath: /etc/curvine # 配置文件存放目录 + command: [ "gomplate" ] + args: [ "-d", "config=$(FLUID_RUNTIME_CONFIG_PATH)", "-f", "/templates/cluster.toml", "-o", "/etc/curvine/curvine.toml" ] + containers: + - name: master + image: "curvine/curvine:latest" + command: + - /entrypoint.sh + args: + - master + - start + env: + # /entrypoint.sh 不支持参数指定配置文件,支撑环境变量配置,默认/app/curvine/conf/curvine-cluster.toml + - name: CURVINE_CONF_FILE + value: "/etc/curvine/curvine.toml" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # curvine checks master hostname which should be one of journal_addrs + - name: CURVINE_MASTER_HOSTNAME + value: "$(POD_NAME).$(FLUID_RUNTIME_COMPONENT_SVC_NAME)" + volumeMounts: + - name: shared-config-volume + mountPath: /etc/curvine # 配置文件存放目录 + - name: curvine-mount-volume + mountPath: /etc/curvine/mount # 配置文件存放目录 + imagePullPolicy: IfNotPresent + volumes: + # emptyDir 共享存储(init容器和主容器互通) + - name: shared-config-volume + emptyDir: { } + - name: curvine-mount-volume + configMap: + name: curvine-mount + defaultMode: 0755 + worker: + service: + headless: { } #需要为worker创建Headless Service + dependencies: + extraResources: + # 使用 extraResources 时,需要定义其挂载路径 + configMaps: + - name: curvine-config + mountPath: "/templates" + template: + spec: + restartPolicy: Always + # 根据 runtime 生成的配置文件 + initContainers: + - name: init-curvine + image: hairyhenderson/gomplate:alpine + # 挂载共享卷到容器内路径 + volumeMounts: + - name: shared-config-volume + mountPath: /etc/curvine # 配置文件存放目录 + command: [ "gomplate" ] + args: [ "-d", "config=$(FLUID_RUNTIME_CONFIG_PATH)", "-f", "/templates/cluster.toml", "-o", "/etc/curvine/curvine.toml" ] + containers: + - name: worker + image: "curvine/curvine:latest" + command: + - /entrypoint.sh + args: + - worker + - start + env: + # /entrypoint.sh 不支持参数指定配置文件,支撑环境变量配置,默认/app/curvine/conf/curvine-cluster.toml + - name: CURVINE_CONF_FILE + value: "/etc/curvine/curvine.toml" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CURVINE_WORKER_HOSTNAME + value: "$(POD_NAME).$(FLUID_RUNTIME_COMPONENT_SVC_NAME)" + volumeMounts: + - name: shared-config-volume + mountPath: /etc/curvine # 配置文件存放目录 + imagePullPolicy: IfNotPresent + volumes: + # emptyDir 共享存储(init容器和主容器互通) + - name: shared-config-volume + emptyDir: { } + client: + dependencies: + # Uncomment to enable secret mount for client (e.g., for JuiceFS FUSE pods) + # secretMount: + # enabled: true + extraResources: + # 使用 extraResources 时,需要定义其挂载路径 + configMaps: + - name: curvine-config + mountPath: "/templates" + template: + spec: + restartPolicy: Always + # 根据 runtime 生成的配置文件 + initContainers: + - name: init-curvine + image: hairyhenderson/gomplate:alpine + # 挂载共享卷到容器内路径 + volumeMounts: + - name: shared-config-volume + mountPath: /etc/curvine # 配置文件存放目录 + command: [ "gomplate" ] + args: [ "-d", "config=$(FLUID_RUNTIME_CONFIG_PATH)", "-f", "/templates/cluster.toml", "-o", "/etc/curvine/curvine.toml" ] + containers: + - name: client + image: "curvine/curvine:latest" + securityContext: #通常client需要配置privileged,用于操作fuse设备 + privileged: true + runAsUser: 0 + command: + - /app/curvine/lib/curvine-fuse + args: + - "--mnt-path" + - "$(FLUID_RUNTIME_MOUNT_PATH)" + - "--conf" + - "/etc/curvine/curvine.toml" + volumeMounts: + - name: shared-config-volume + mountPath: /etc/curvine # 配置文件存放目录 + imagePullPolicy: IfNotPresent + volumes: + # emptyDir 共享存储(init容器和主容器互通) + - name: shared-config-volume + emptyDir: { } diff --git a/test/gha-e2e/cacheruntime-selector/dataset.yaml b/test/gha-e2e/cacheruntime-selector/dataset.yaml new file mode 100644 index 00000000000..fb24bea9506 --- /dev/null +++ b/test/gha-e2e/cacheruntime-selector/dataset.yaml @@ -0,0 +1,31 @@ +apiVersion: data.fluid.io/v1alpha1 +kind: Dataset +metadata: + name: selector-demo +spec: + accessModes: ["ReadWriteMany"] + mounts: + - mountPoint: "s3://test" + # path: /minio + name: minio + # bin/cv mount s3://test /minio \ + # -c s3.endpoint_url=http://127.0.0.1:19000 \ + # -c s3.region_name=us-east-1 \ + # -c s3.path_style=true \ + # -c s3.credentials.access=minioadmin \ + # -c s3.credentials.secret=minioadmin + options: + endpoint_url: "http://minio:9000" + region_name: "us-east-1" + path_style: "true" + encryptOptions: + - name: access + valueFrom: + secretKeyRef: + name: curvine-secret + key: access-key + - name: secret + valueFrom: + secretKeyRef: + name: curvine-secret + key: secret-key diff --git a/test/gha-e2e/cacheruntime-selector/minio.yaml b/test/gha-e2e/cacheruntime-selector/minio.yaml new file mode 100644 index 00000000000..c2ace511a74 --- /dev/null +++ b/test/gha-e2e/cacheruntime-selector/minio.yaml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: Secret +metadata: + name: curvine-secret +stringData: + access-key: minioadmin + secret-key: minioadmin +--- +apiVersion: v1 +kind: Service +metadata: + name: minio +spec: + type: ClusterIP + ports: + - port: 9000 + targetPort: 9000 + protocol: TCP + selector: + app: minio +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + # This name uniquely identifies the Deployment + name: minio +spec: + selector: + matchLabels: + app: minio + strategy: + type: Recreate + template: + metadata: + labels: + # Label is used as selector in the service. + app: minio + spec: + containers: + - name: minio + # Pulls the default Minio image from Docker Hub + image: minio/minio + imagePullPolicy: IfNotPresent + resources: + limits: + memory: "512Mi" + args: + - server + - /data + env: + # Minio access key and secret key + - name: MINIO_ROOT_USER + value: "minioadmin" + - name: MINIO_ROOT_PASSWORD + value: "minioadmin" + ports: + - containerPort: 9000 + hostPort: 9000 + automountServiceAccountToken: false diff --git a/test/gha-e2e/cacheruntime-selector/minio_create_bucket.yaml b/test/gha-e2e/cacheruntime-selector/minio_create_bucket.yaml new file mode 100644 index 00000000000..f2e0f171597 --- /dev/null +++ b/test/gha-e2e/cacheruntime-selector/minio_create_bucket.yaml @@ -0,0 +1,26 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-bucket-create +spec: + template: + spec: + containers: + - name: mc + image: minio/mc + imagePullPolicy: IfNotPresent + resources: + limits: + memory: "512Mi" + command: + - /bin/sh + - -c + - "mc alias set myminio http://minio:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD && mc mb myminio/test" + env: + # Minio access key and secret key + - name: MINIO_ROOT_USER + value: "minioadmin" + - name: MINIO_ROOT_PASSWORD + value: "minioadmin" + restartPolicy: OnFailure + backoffLimit: 4 diff --git a/test/gha-e2e/cacheruntime-selector/mount.yaml b/test/gha-e2e/cacheruntime-selector/mount.yaml new file mode 100644 index 00000000000..2a8a6049f9d --- /dev/null +++ b/test/gha-e2e/cacheruntime-selector/mount.yaml @@ -0,0 +1,146 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: curvine-mount +data: + # 挂载底层文件系统的脚本(仅限测试场景) + mountUfs.sh: | + #!/bin/bash + # 1. 原始JSON + json=$(cat ${FLUID_RUNTIME_CONFIG_PATH}) + # 2. 提取mounts数组内部内容 + mounts_raw=$(echo "$json" | sed -E 's/.*"mounts":\[([^]]+)\].*/\1/') + + # ====================标准兼容的数组拆分(核心!) ==================== + # 替换},\s{ 为标准正则, 兼容所有POSIX sed, 自动处理空格 + mounts_items=$(echo "$mounts_raw" \ + | sed 's/},[[:space:]]*{/\n/g' \ + | sed 's/^{//; s/}$//; s/^[[:space:]]*//g; s/[[:space:]]*$//g') + + # 3. 遍历解析 + index=0 + echo "$mounts_items" | while IFS= read -r item; do + #echo "第 $index 个 mounts 元素" + index=$((index + 1)) + + # 提取基础字段 + mountPoint=$(echo "$item" | sed -nE 's/.*"mountPoint":"([^"]+)".*/\1/p') + path=$(echo "$item" | sed -nE 's/.*"path":"([^"]+)".*/\1/p') + + mounted=$(/app/curvine/bin/cv mount) + if echo "$mounted" | grep "$mountPoint" >/dev/null 2>&1; then + continue; + fi + + [ -z "$mountPoint" ] && { echo "mountPoint is not set or empty" >&2; exit 1; } + [ -z "$path" ] && { echo "path is not set or empty" >&2; exit 1; } + + # ==================== 字段提取加p 仅输出匹配结果 ==================== + # Extract encryptOptions paths from the JSON + encryptOptions_raw=$(echo "$item" | sed -nE 's/.*"encryptOptions":\{([^}]+)\}.*/\1/p') + + # Extract access and secret file paths from encryptOptions + access_path="" + secret_path="" + if [ -n "$encryptOptions_raw" ]; then + access_path=$(echo "$encryptOptions_raw" | sed -nE 's/.*"access":"([^"]+)".*/\1/p') + secret_path=$(echo "$encryptOptions_raw" | sed -nE 's/.*"secret":"([^"]+)".*/\1/p') + fi + + # Read actual values from secret files if paths are provided + access="" + secret="" + if [ -n "$access_path" ] && [ -f "$access_path" ]; then + access=$(cat "$access_path") + fi + if [ -n "$secret_path" ] && [ -f "$secret_path" ]; then + secret=$(cat "$secret_path") + fi + + endpoint=$(echo "$item" | sed -nE 's/.*"endpoint_url":"([^"]+)".*/\1/p') + region=$(echo "$item" | sed -nE 's/.*"region_name":"([^"]+)".*/\1/p') + path_style=$(echo "$item" | sed -nE 's/.*"path_style":"([^"]+)".*/\1/p') + + # 打印调试(缺失字段会显示空,正确) + #echo "access: $access" + #echo "region: $region" + #echo "endpoint: $endpoint" + #echo "path_style: $path_style" + #echo "secret: $secret" + + # 必填参数校验 + CV_PARAMS="" + [ -z "$endpoint" ] && { echo "endpoint option is not set or empty" >&2; exit 1; } + [ -z "$access" ] && { echo "access option is not set or empty" >&2; exit 1; } + [ -z "$secret" ] && { echo "secret option is not set or empty" >&2; exit 1; } + + # 拼接必填参数 + CV_PARAMS="$CV_PARAMS -c s3.endpoint_url=$endpoint" + CV_PARAMS="$CV_PARAMS -c s3.credentials.access=$access" + CV_PARAMS="$CV_PARAMS -c s3.credentials.secret=$secret" + + # 可选参数:缺失自动跳过 + [ -n "$region" ] && CV_PARAMS="$CV_PARAMS -c s3.region_name=$region" + [ -n "$path_style" ] && CV_PARAMS="$CV_PARAMS -c s3.path_style=$path_style" + + # 最终命令 + CMD="/app/curvine/bin/cv mount $mountPoint $path --check-path-consist false $CV_PARAMS" + #echo "执行命令:$CMD" + + eval "$CMD" > /dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "mount $mountPoint failed" >&2 + exit 1 + fi + done + + # /app/curvine/bin/cv mount 的输出如下所示 + # Mount Table: + # +-------------+----------------+-------------+-------------+-------------------+ + # | ID | Curvine Path | UFS Path | Write Type | Read Verify UFS | + # +-------------+----------------+-------------+-------------+-------------------+ + # | 1664140379 | /minio | s3://test/ | fs_mode | no | + # +-------------+----------------+-------------+-------------+-------------------+ + # Total mount points: 1 + + # 获取所有已挂载的 Curvine Path 并组装成 CacheRuntimeMountUfsOutput 格式的 JSON + getCurvinePathsAsJson() { + local output + output=$(/app/curvine/bin/cv mount) + + # 提取表格中的数据行(跳过表头和分隔线) + local paths=() + while IFS= read -r line; do + # 跳过空行、表头行和分隔线 + if [[ -z "$line" ]] || [[ "$line" == *"|"*"ID"* ]] || [[ "$line" == *"+"*"-"*"+"* ]] || [[ ! "$line" =~ ^\| ]] ; then + continue + fi + + # 提取第二列(Curvine Path)的值 + # 格式: | ID | Curvine Path | UFS Path | Write Type | Read Verify UFS | + local path + path=$(echo "$line" | sed -E 's/^\|[[:space:]]*[^|]+\|[[:space:]]*([^|]+)\|.*/\1/' | sed 's/[[:space:]]*$//') + + if [[ -n "$path" ]]; then + paths+=("$path") + fi + done <<< "$output" + + # 组装成 CacheRuntimeMountUfsOutput 格式的 JSON: {"mounted": ["path1", "path2", ...]} + local json='{"mounted":[' + for i in "${!paths[@]}"; do + if [ $i -gt 0 ]; then + json+="," + fi + # 转义特殊字符并添加到 JSON 数组 + local escaped_path + escaped_path=$(echo "${paths[$i]}" | sed 's/\\/\\\\/g; s/"/\\"/g') + json+="\"${escaped_path}\"" + done + json+=']}' + echo "$json" + } + + # 调用函数获取 CacheRuntimeMountUfsOutput 格式的 JSON + CURVINE_PATHS_JSON=$(getCurvinePathsAsJson) + echo "$CURVINE_PATHS_JSON" diff --git a/test/gha-e2e/cacheruntime-selector/test.sh b/test/gha-e2e/cacheruntime-selector/test.sh new file mode 100644 index 00000000000..271446901b2 --- /dev/null +++ b/test/gha-e2e/cacheruntime-selector/test.sh @@ -0,0 +1,209 @@ +#!/bin/bash +testname="cache runtime status.selector e2e" +dataset_name="selector-demo" +bucket_create_job_name="minio-bucket-create" + +function syslog() { + echo ">>> $1" +} + +function panic() { + local err_msg=$1 + syslog "test \"$testname\" failed: $err_msg" + exit 1 +} + +function setup() { + kubectl create -f test/gha-e2e/cacheruntime-selector/minio.yaml + kubectl create -f test/gha-e2e/cacheruntime-selector/minio_create_bucket.yaml + wait_job_completed "$bucket_create_job_name" + kubectl create -f test/gha-e2e/cacheruntime-selector/mount.yaml +} + +function create_dataset() { + kubectl create -f test/gha-e2e/cacheruntime-selector/cacheruntimeclass.yaml + kubectl create -f test/gha-e2e/cacheruntime-selector/dataset.yaml + kubectl create -f test/gha-e2e/cacheruntime-selector/cacheruntime.yaml + if [[ -z "$(kubectl get dataset $dataset_name -oname)" ]]; then + panic "failed to create dataset $dataset_name" + fi + if [[ -z "$(kubectl get cacheruntime $dataset_name -oname)" ]]; then + panic "failed to create cache runtime $dataset_name" + fi + if [[ -z "$(kubectl get cacheruntimeclass $dataset_name -oname)" ]]; then + panic "failed to create cache runtime class $dataset_name" + fi +} + +function wait_dataset_bound() { + local deadline=600 + local last_state="" + local log_interval=0 + local log_times=0 + while true; do + last_state=$(kubectl get dataset $dataset_name -ojsonpath='{@.status.phase}') + if [[ $log_interval -eq 3 ]]; then + log_times=$((log_times + 1)) + syslog "checking dataset.status.phase==Bound (already $((log_times * log_interval * 5))s, last state: $last_state)" + if [[ $((log_times * log_interval * 5)) -ge $deadline ]]; then + panic "timeout for ${deadline}s!" + fi + log_interval=0 + fi + if [[ "$last_state" == "Bound" ]]; then + break + fi + log_interval=$((log_interval + 1)) + sleep 5 + done + syslog "Found dataset $dataset_name status.phase==Bound" +} + +function wait_cache_worker_ready() { + local deadline=180 + local worker_component_name="${dataset_name}-worker" + local worker_selector="cacheruntime.fluid.io/component-name=${worker_component_name}" + local last_phase="" + local runtime_ready_replicas="" + local runtime_desired_replicas="" + local worker_pod="" + local worker_registered="false" + local pod_states="" + local log_interval=0 + local log_times=0 + while true; do + last_phase=$(kubectl get cacheruntime "$dataset_name" -ojsonpath='{@.status.worker.phase}') + runtime_ready_replicas=$(kubectl get cacheruntime "$dataset_name" -ojsonpath='{@.status.worker.readyReplicas}') + runtime_desired_replicas=$(kubectl get cacheruntime "$dataset_name" -ojsonpath='{@.status.worker.desiredReplicas}') + worker_pod=$(kubectl get pod -l "$worker_selector" -ojsonpath='{.items[0].metadata.name}' 2>/dev/null) + worker_registered="false" + if [[ -n "$worker_pod" ]] && kubectl logs "$worker_pod" -c worker --tail=200 2>/dev/null | grep -q "worker register success"; then + worker_registered="true" + fi + pod_states=$(kubectl get pod -l "$worker_selector" -ojsonpath='{range .items[*]}{.metadata.name}:{range .status.containerStatuses[*]}{.ready}{end}:{.status.phase}{" "}{end}' 2>/dev/null) + if [[ $log_interval -eq 3 ]]; then + log_times=$((log_times + 1)) + syslog "checking cache worker readiness (already $((log_times * log_interval * 5))s, runtime phase: ${last_phase:-}, runtime ready/desired: ${runtime_ready_replicas:-}/${runtime_desired_replicas:-}, registered: ${worker_registered}, pods: ${pod_states:-})" + if [[ $((log_times * log_interval * 5)) -ge $deadline ]]; then + panic "timeout waiting for cache worker pod ready after ${deadline}s" + fi + log_interval=0 + fi + if [[ "$last_phase" == "Ready" ]] && \ + [[ -n "$runtime_desired_replicas" ]] && \ + [[ "$runtime_desired_replicas" != "0" ]] && \ + [[ "$runtime_ready_replicas" == "$runtime_desired_replicas" ]] && \ + kubectl wait --for=condition=Ready --timeout=5s pod -l "$worker_selector" >/dev/null 2>&1 && \ + [[ "$worker_registered" == "true" ]]; then + break + fi + log_interval=$((log_interval + 1)) + sleep 5 + done + syslog "Found ready cache worker pod for $dataset_name" +} + +function wait_job_completed() { + local job_name=$1 + local succeed="" + local deadline=600 + local counter=0 + local job_failed="" + while true; do + succeed=$(kubectl get job "$job_name" -ojsonpath='{@.status.succeeded}') + [[ -z "$succeed" ]] && succeed=0 + if [[ "$succeed" -ge "1" ]]; then + break + fi + job_failed=$(kubectl get job "$job_name" \ + -ojsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true) + if [[ "$job_failed" == "True" ]]; then + panic "job $job_name failed (all retries exhausted)" + fi + counter=$((counter + 1)) + if [[ $((counter * 5)) -ge $deadline ]]; then + panic "timeout ${deadline}s waiting for job $job_name to complete" + fi + sleep 5 + done + syslog "Found succeeded job $job_name" +} + +function assert_status_selector() { + local selector + selector=$(kubectl get cacheruntime "$dataset_name" -ojsonpath='{@.status.selector}') + syslog "cacheruntime $dataset_name status.selector = ${selector:-}" + + if [[ -z "$selector" ]]; then + panic "status.selector is empty" + fi + + local expected="cacheruntime.fluid.io/component-name=${dataset_name}-worker,cacheruntime.fluid.io/name=${dataset_name}" + if [[ "$selector" != "$expected" ]]; then + panic "status.selector mismatch: got '$selector', expected '$expected'" + fi + syslog "status.selector matches expected value" +} + +function assert_selector_resolves_worker_pods() { + local selector + selector=$(kubectl get cacheruntime "$dataset_name" -ojsonpath='{@.status.selector}') + + local pod_count + pod_count=$(kubectl get pod -l "$selector" --no-headers 2>/dev/null | wc -l) + if [[ "$pod_count" -lt 1 ]]; then + panic "no worker pods found via selector '$selector'" + fi + syslog "selector '$selector' correctly resolved to $pod_count worker pod(s)" +} + +function assert_scale_subresource() { + local status_selector + status_selector=$(kubectl get cacheruntime "$dataset_name" -ojsonpath='{@.status.selector}') + + local scale_json + scale_json=$(kubectl get --raw "/apis/data.fluid.io/v1alpha1/namespaces/default/cacheruntimes/${dataset_name}/scale" 2>/dev/null) + if [[ -z "$scale_json" ]]; then + panic "failed to read scale subresource for cacheruntime $dataset_name" + fi + syslog "scale subresource: $scale_json" + + local scale_selector + scale_selector=$(echo "$scale_json" | jq -r '.status.selector') + if [[ "$scale_selector" != "$status_selector" ]]; then + panic "scale subresource selector '$scale_selector' does not match status.selector '$status_selector'" + fi + syslog "scale subresource selector matches status.selector: $scale_selector" + + local scale_replicas + scale_replicas=$(echo "$scale_json" | jq -r '.status.replicas') + if [[ -z "$scale_replicas" || "$scale_replicas" == "null" || "$scale_replicas" -lt 1 ]]; then + panic "scale subresource status.replicas is invalid: '$scale_replicas'" + fi + syslog "scale subresource status.replicas: $scale_replicas" +} + +function dump_env_and_clean_up() { + syslog "Cleaning up resources for testcase $testname" + kubectl delete --ignore-not-found -f test/gha-e2e/cacheruntime-selector/dataset.yaml + kubectl delete --ignore-not-found -f test/gha-e2e/cacheruntime-selector/cacheruntime.yaml + kubectl delete --ignore-not-found -f test/gha-e2e/cacheruntime-selector/cacheruntimeclass.yaml + kubectl delete --ignore-not-found -f test/gha-e2e/cacheruntime-selector/minio.yaml + kubectl delete --ignore-not-found -f test/gha-e2e/cacheruntime-selector/mount.yaml + kubectl delete --ignore-not-found -f test/gha-e2e/cacheruntime-selector/minio_create_bucket.yaml +} + +function main() { + syslog "[TESTCASE $testname STARTS AT $(date)]" + trap dump_env_and_clean_up EXIT + setup + create_dataset + wait_dataset_bound + wait_cache_worker_ready + assert_status_selector + assert_selector_resolves_worker_pods + assert_scale_subresource + syslog "[TESTCASE $testname SUCCEEDED AT $(date)]" +} + +main \ No newline at end of file