From bcd7538ae39ddf93adeabac95bc4918d1574c02d Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 2 Jan 2026 13:58:23 +0000 Subject: [PATCH 1/2] chore: Migrate gsutil usage to gcloud storage --- .kokoro/trampoline_v2.sh | 4 ++-- cloudbuild.yaml | 24 ++++++++++++------- java/.ci/Jenkinsfile | 10 ++++---- java/.ci/UiJenkinsfile | 4 ++-- java/bin/start.sh | 7 +++--- .../dataproc/templates/dataplex/README.md | 4 ++-- .../dataproc/templates/general/README.md | 3 +-- .../vertex_pipeline_pyspark.ipynb | 4 ++-- python/.ci/python-integration-tests.yaml | 6 ++--- python/bin/start.sh | 6 ++--- 10 files changed, 38 insertions(+), 34 deletions(-) diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh index 4af6cdc26..ac01c22f2 100644 --- a/.kokoro/trampoline_v2.sh +++ b/.kokoro/trampoline_v2.sh @@ -26,8 +26,8 @@ # To run this script, first download few files from gcs to /dev/shm. # (/dev/shm is passed into the container as KOKORO_GFILE_DIR). # -# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/secrets_viewer_service_account.json /dev/shm -# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/automl_secrets.txt /dev/shm +# gcloud storage cp gs://cloud-devrel-kokoro-resources/python-docs-samples/secrets_viewer_service_account.json /dev/shm +# gcloud storage cp gs://cloud-devrel-kokoro-resources/python-docs-samples/automl_secrets.txt /dev/shm # # Then run the script. # .kokoro/trampoline_v2.sh diff --git a/cloudbuild.yaml b/cloudbuild.yaml index bd09967fa..cc2f232a2 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -26,29 +26,33 @@ steps: - '--output=${_BINARY_PREFIX}.egg' dir: '${_REPO}/python' id: build-python-egg - - name: gcr.io/cloud-builders/gsutil + - name: gcr.io/cloud-builders/gcloud entrypoint: 'bash' args: - '-c' - | - gsutil cp ${_REPO}/python/main.py gs://${_PUBLIC_BUCKET}/$(date -u +%Y-%m-%d)_$TAG_NAME/python/ + gcloud storage cp ${_REPO}/python/main.py gs://${_PUBLIC_BUCKET}/$(date -u +%Y-%m-%d)_$TAG_NAME/python/ id: cp-python-main-to-gcs-tag-folder - - name: gcr.io/cloud-builders/gsutil + - name: gcr.io/cloud-builders/gcloud + entrypoint: gcloud args: + - storage - cp - main.py - 'gs://${_PUBLIC_BUCKET}/latest/python/' dir: '${_REPO}/python' id: cp-python-main-to-gcs-latest-folder - - name: gcr.io/cloud-builders/gsutil + - name: gcr.io/cloud-builders/gcloud entrypoint: 'bash' args: - '-c' - | - gsutil cp ${_REPO}/python/${_BINARY_PREFIX}.egg gs://${_PUBLIC_BUCKET}/$(date -u +%Y-%m-%d)_$TAG_NAME/python/ + gcloud storage cp ${_REPO}/python/${_BINARY_PREFIX}.egg gs://${_PUBLIC_BUCKET}/$(date -u +%Y-%m-%d)_$TAG_NAME/python/ id: cp-python-egg-to-gcs-tag-folder - - name: gcr.io/cloud-builders/gsutil + - name: gcr.io/cloud-builders/gcloud + entrypoint: gcloud args: + - storage - cp - '${_BINARY_PREFIX}.egg' - 'gs://${_PUBLIC_BUCKET}/latest/python/' @@ -61,15 +65,17 @@ steps: dir: '${_REPO}/java' id: build-java-jar entrypoint: mvn - - name: gcr.io/cloud-builders/gsutil + - name: gcr.io/cloud-builders/gcloud entrypoint: 'bash' args: - '-c' - | - gsutil cp ${_REPO}/java/target/dataproc-templates-1.0-SNAPSHOT.jar gs://${_PUBLIC_BUCKET}/$(date -u +%Y-%m-%d)_$TAG_NAME/java/ + gcloud storage cp ${_REPO}/java/target/dataproc-templates-1.0-SNAPSHOT.jar gs://${_PUBLIC_BUCKET}/$(date -u +%Y-%m-%d)_$TAG_NAME/java/ id: cp-java-egg-to-gcs-tag-folder - - name: gcr.io/cloud-builders/gsutil + - name: gcr.io/cloud-builders/gcloud + entrypoint: gcloud args: + - storage - cp - dataproc-templates-1.0-SNAPSHOT.jar - 'gs://${_PUBLIC_BUCKET}/latest/java/${_BINARY_PREFIX}.jar' diff --git a/java/.ci/Jenkinsfile b/java/.ci/Jenkinsfile index 0a8c32abe..3bf533270 100644 --- a/java/.ci/Jenkinsfile +++ b/java/.ci/Jenkinsfile @@ -68,9 +68,9 @@ pipeline { steps { catchError { sh ''' - gsutil rm -r gs://dataproc-templates/integration-testing/checkpoint/KAFKATOBQ 2> /dev/null || true - gsutil rm -r gs://dataproc-templates/integration-testing/output/KAFKATOGCS 2> /dev/null || true - gsutil rm -r gs://dataproc-templates/integration-testing/output/KAFKATOGCS_DStream 2> /dev/null || true + gcloud storage rm --recursive gs://dataproc-templates/integration-testing/checkpoint/KAFKATOBQ 2> /dev/null || true + gcloud storage rm --recursive gs://dataproc-templates/integration-testing/output/KAFKATOGCS 2> /dev/null || true + gcloud storage rm --recursive gs://dataproc-templates/integration-testing/output/KAFKATOGCS_DStream 2> /dev/null || true cat > /tmp/employeecatalog.json << EOF { @@ -86,7 +86,7 @@ cat > /tmp/employeecatalog.json << EOF EOF cat /tmp/employeecatalog.json - gsutil cp /tmp/employeecatalog.json gs://dataproc-templates/conf/ + gcloud storage cp /tmp/employeecatalog.json gs://dataproc-templates/conf/ ''' } @@ -193,7 +193,7 @@ EOF catchError { sh ''' if [ $GIT_BRANCH_LOCAL != "main" ];then - gsutil rm -r $GCS_STAGING_LOCATION 2> /dev/null || true + gcloud storage rm --recursive $GCS_STAGING_LOCATION 2> /dev/null || true fi ''' } diff --git a/java/.ci/UiJenkinsfile b/java/.ci/UiJenkinsfile index ffb776032..d30b8b0fd 100644 --- a/java/.ci/UiJenkinsfile +++ b/java/.ci/UiJenkinsfile @@ -99,7 +99,7 @@ pipeline { LOCAL_JAR_Path=./target/${JAR_FILE} - gsutil cp ${LOCAL_JAR_Path} ${jarPath} + gcloud storage cp ${LOCAL_JAR_Path} ${jarPath} ''' } @@ -121,7 +121,7 @@ pipeline { catchError { sh ''' if [ $GIT_BRANCH_LOCAL != "main" ];then - gsutil rm -r $jarPath 2> /dev/null || true + gcloud storage rm --recursive $jarPath 2> /dev/null || true fi ''' } diff --git a/java/bin/start.sh b/java/bin/start.sh index e77904643..9b0d21c43 100755 --- a/java/bin/start.sh +++ b/java/bin/start.sh @@ -58,7 +58,7 @@ if [ -z "$SKIP_BUILD" ]; then #Copy jar file to GCS bucket Staging folder echo_formatted "Copying ${PROJECT_ROOT_DIR}/target/${JAR_FILE} to staging bucket: ${GCS_STAGING_LOCATION}/${JAR_FILE}" - gsutil cp ${PROJECT_ROOT_DIR}/target/${JAR_FILE} ${GCS_STAGING_LOCATION}/${JAR_FILE} + gcloud storage cp ${PROJECT_ROOT_DIR}/target/${JAR_FILE} ${GCS_STAGING_LOCATION}/${JAR_FILE} check_status $? "\n Commands to copy the project jar file to GCS Staging location went fine, thus we are good to go \n" "\n It seems like there is some issue in copying the project jar file to GCS Staging location \n" @@ -122,8 +122,8 @@ if [ -n "${CATALOG}" ]; then echo "Downloading Hbase jar dependency" wget https://repo1.maven.org/maven2/org/apache/hbase/hbase-client/2.4.12/hbase-client-2.4.12.jar wget https://repo1.maven.org/maven2/org/apache/hbase/hbase-shaded-mapreduce/2.4.12/hbase-shaded-mapreduce-2.4.12.jar - gsutil copy hbase-client-2.4.12.jar ${GCS_STAGING_LOCATION}/hbase-client-2.4.12.jar - gsutil copy hbase-shaded-mapreduce-2.4.12.jar ${GCS_STAGING_LOCATION}/hbase-shaded-mapreduce-2.4.12.jar + gcloud storage cp hbase-client-2.4.12.jar ${GCS_STAGING_LOCATION}/hbase-client-2.4.12.jar + gcloud storage cp hbase-shaded-mapreduce-2.4.12.jar ${GCS_STAGING_LOCATION}/hbase-shaded-mapreduce-2.4.12.jar echo "Passing downloaded dependency jars" OPT_JARS="${OPT_JARS},${GCS_STAGING_LOCATION}/hbase-client-2.4.12.jar,${GCS_STAGING_LOCATION}/hbase-shaded-mapreduce-2.4.12.jar" rm hbase-client-2.4.12.jar @@ -186,4 +186,3 @@ check_status $spark_status "\n Spark Command ran successful \n" "\n It seems lik echo "We will love to hear your feedback at: https://forms.gle/XXCJeWeCJJ9fNLQS6" echo "Email us at: dataproc-templates-support-external@googlegroups.com" - diff --git a/java/src/main/java/com/google/cloud/dataproc/templates/dataplex/README.md b/java/src/main/java/com/google/cloud/dataproc/templates/dataplex/README.md index 91c667a86..53fb8c5f1 100644 --- a/java/src/main/java/com/google/cloud/dataproc/templates/dataplex/README.md +++ b/java/src/main/java/com/google/cloud/dataproc/templates/dataplex/README.md @@ -11,8 +11,8 @@ from Cloud Storage and the target BQ table will be overwritten. Download jar and properties file ``` export GCS_STAGING_LOCATION=gs://bucket/path/to/staging/folder/ -gsutil -u cp gs://dataplex-dataproc-templates-artifacts/dataproc-templates-1.0-SNAPSHOT.jar ${GCS_STAGING_LOCATION} -gsutil -u cp gs://dataplex-dataproc-templates-artifacts/log4j-spark-driver-template.properties ${GCS_STAGING_LOCATION} +gcloud storage cp gs://dataplex-dataproc-templates-artifacts/dataproc-templates-1.0-SNAPSHOT.jar ${GCS_STAGING_LOCATION} +gcloud storage cp gs://dataplex-dataproc-templates-artifacts/log4j-spark-driver-template.properties ${GCS_STAGING_LOCATION} ``` Create Dataplex Task diff --git a/java/src/main/java/com/google/cloud/dataproc/templates/general/README.md b/java/src/main/java/com/google/cloud/dataproc/templates/general/README.md index 4d929d51f..c472141e1 100644 --- a/java/src/main/java/com/google/cloud/dataproc/templates/general/README.md +++ b/java/src/main/java/com/google/cloud/dataproc/templates/general/README.md @@ -66,7 +66,7 @@ export SUBNET=projects//regions//subnetworks/ Date: Fri, 2 Jan 2026 20:42:46 +0530 Subject: [PATCH 2/2] Update README.md --- .../com/google/cloud/dataproc/templates/dataplex/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/java/src/main/java/com/google/cloud/dataproc/templates/dataplex/README.md b/java/src/main/java/com/google/cloud/dataproc/templates/dataplex/README.md index 53fb8c5f1..27af6d1f8 100644 --- a/java/src/main/java/com/google/cloud/dataproc/templates/dataplex/README.md +++ b/java/src/main/java/com/google/cloud/dataproc/templates/dataplex/README.md @@ -11,8 +11,8 @@ from Cloud Storage and the target BQ table will be overwritten. Download jar and properties file ``` export GCS_STAGING_LOCATION=gs://bucket/path/to/staging/folder/ -gcloud storage cp gs://dataplex-dataproc-templates-artifacts/dataproc-templates-1.0-SNAPSHOT.jar ${GCS_STAGING_LOCATION} -gcloud storage cp gs://dataplex-dataproc-templates-artifacts/log4j-spark-driver-template.properties ${GCS_STAGING_LOCATION} +gcloud storage cp gs://dataplex-dataproc-templates-artifacts/dataproc-templates-1.0-SNAPSHOT.jar ${GCS_STAGING_LOCATION} --billing-project= +gcloud storage cp gs://dataplex-dataproc-templates-artifacts/log4j-spark-driver-template.properties ${GCS_STAGING_LOCATION} --billing-project= ``` Create Dataplex Task @@ -101,4 +101,4 @@ FROM __table__ WHERE id > 100 -``` \ No newline at end of file +```