From bcd7538ae39ddf93adeabac95bc4918d1574c02d Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 2 Jan 2026 13:58:23 +0000 Subject: [PATCH 1/4] chore: Migrate gsutil usage to gcloud storage --- .kokoro/trampoline_v2.sh | 4 ++-- cloudbuild.yaml | 24 ++++++++++++------- java/.ci/Jenkinsfile | 10 ++++---- java/.ci/UiJenkinsfile | 4 ++-- java/bin/start.sh | 7 +++--- .../dataproc/templates/dataplex/README.md | 4 ++-- .../dataproc/templates/general/README.md | 3 +-- .../vertex_pipeline_pyspark.ipynb | 4 ++-- python/.ci/python-integration-tests.yaml | 6 ++--- python/bin/start.sh | 6 ++--- 10 files changed, 38 insertions(+), 34 deletions(-) diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh index 4af6cdc26..ac01c22f2 100644 --- a/.kokoro/trampoline_v2.sh +++ b/.kokoro/trampoline_v2.sh @@ -26,8 +26,8 @@ # To run this script, first download few files from gcs to /dev/shm. # (/dev/shm is passed into the container as KOKORO_GFILE_DIR). # -# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/secrets_viewer_service_account.json /dev/shm -# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/automl_secrets.txt /dev/shm +# gcloud storage cp gs://cloud-devrel-kokoro-resources/python-docs-samples/secrets_viewer_service_account.json /dev/shm +# gcloud storage cp gs://cloud-devrel-kokoro-resources/python-docs-samples/automl_secrets.txt /dev/shm # # Then run the script. # .kokoro/trampoline_v2.sh diff --git a/cloudbuild.yaml b/cloudbuild.yaml index bd09967fa..cc2f232a2 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -26,29 +26,33 @@ steps: - '--output=${_BINARY_PREFIX}.egg' dir: '${_REPO}/python' id: build-python-egg - - name: gcr.io/cloud-builders/gsutil + - name: gcr.io/cloud-builders/gcloud entrypoint: 'bash' args: - '-c' - | - gsutil cp ${_REPO}/python/main.py gs://${_PUBLIC_BUCKET}/$(date -u +%Y-%m-%d)_$TAG_NAME/python/ + gcloud storage cp ${_REPO}/python/main.py gs://${_PUBLIC_BUCKET}/$(date -u +%Y-%m-%d)_$TAG_NAME/python/ id: cp-python-main-to-gcs-tag-folder - - name: gcr.io/cloud-builders/gsutil + - name: gcr.io/cloud-builders/gcloud + entrypoint: gcloud args: + - storage - cp - main.py - 'gs://${_PUBLIC_BUCKET}/latest/python/' dir: '${_REPO}/python' id: cp-python-main-to-gcs-latest-folder - - name: gcr.io/cloud-builders/gsutil + - name: gcr.io/cloud-builders/gcloud entrypoint: 'bash' args: - '-c' - | - gsutil cp ${_REPO}/python/${_BINARY_PREFIX}.egg gs://${_PUBLIC_BUCKET}/$(date -u +%Y-%m-%d)_$TAG_NAME/python/ + gcloud storage cp ${_REPO}/python/${_BINARY_PREFIX}.egg gs://${_PUBLIC_BUCKET}/$(date -u +%Y-%m-%d)_$TAG_NAME/python/ id: cp-python-egg-to-gcs-tag-folder - - name: gcr.io/cloud-builders/gsutil + - name: gcr.io/cloud-builders/gcloud + entrypoint: gcloud args: + - storage - cp - '${_BINARY_PREFIX}.egg' - 'gs://${_PUBLIC_BUCKET}/latest/python/' @@ -61,15 +65,17 @@ steps: dir: '${_REPO}/java' id: build-java-jar entrypoint: mvn - - name: gcr.io/cloud-builders/gsutil + - name: gcr.io/cloud-builders/gcloud entrypoint: 'bash' args: - '-c' - | - gsutil cp ${_REPO}/java/target/dataproc-templates-1.0-SNAPSHOT.jar gs://${_PUBLIC_BUCKET}/$(date -u +%Y-%m-%d)_$TAG_NAME/java/ + gcloud storage cp ${_REPO}/java/target/dataproc-templates-1.0-SNAPSHOT.jar gs://${_PUBLIC_BUCKET}/$(date -u +%Y-%m-%d)_$TAG_NAME/java/ id: cp-java-egg-to-gcs-tag-folder - - name: gcr.io/cloud-builders/gsutil + - name: gcr.io/cloud-builders/gcloud + entrypoint: gcloud args: + - storage - cp - dataproc-templates-1.0-SNAPSHOT.jar - 'gs://${_PUBLIC_BUCKET}/latest/java/${_BINARY_PREFIX}.jar' diff --git a/java/.ci/Jenkinsfile b/java/.ci/Jenkinsfile index 0a8c32abe..3bf533270 100644 --- a/java/.ci/Jenkinsfile +++ b/java/.ci/Jenkinsfile @@ -68,9 +68,9 @@ pipeline { steps { catchError { sh ''' - gsutil rm -r gs://dataproc-templates/integration-testing/checkpoint/KAFKATOBQ 2> /dev/null || true - gsutil rm -r gs://dataproc-templates/integration-testing/output/KAFKATOGCS 2> /dev/null || true - gsutil rm -r gs://dataproc-templates/integration-testing/output/KAFKATOGCS_DStream 2> /dev/null || true + gcloud storage rm --recursive gs://dataproc-templates/integration-testing/checkpoint/KAFKATOBQ 2> /dev/null || true + gcloud storage rm --recursive gs://dataproc-templates/integration-testing/output/KAFKATOGCS 2> /dev/null || true + gcloud storage rm --recursive gs://dataproc-templates/integration-testing/output/KAFKATOGCS_DStream 2> /dev/null || true cat > /tmp/employeecatalog.json << EOF { @@ -86,7 +86,7 @@ cat > /tmp/employeecatalog.json << EOF EOF cat /tmp/employeecatalog.json - gsutil cp /tmp/employeecatalog.json gs://dataproc-templates/conf/ + gcloud storage cp /tmp/employeecatalog.json gs://dataproc-templates/conf/ ''' } @@ -193,7 +193,7 @@ EOF catchError { sh ''' if [ $GIT_BRANCH_LOCAL != "main" ];then - gsutil rm -r $GCS_STAGING_LOCATION 2> /dev/null || true + gcloud storage rm --recursive $GCS_STAGING_LOCATION 2> /dev/null || true fi ''' } diff --git a/java/.ci/UiJenkinsfile b/java/.ci/UiJenkinsfile index ffb776032..d30b8b0fd 100644 --- a/java/.ci/UiJenkinsfile +++ b/java/.ci/UiJenkinsfile @@ -99,7 +99,7 @@ pipeline { LOCAL_JAR_Path=./target/${JAR_FILE} - gsutil cp ${LOCAL_JAR_Path} ${jarPath} + gcloud storage cp ${LOCAL_JAR_Path} ${jarPath} ''' } @@ -121,7 +121,7 @@ pipeline { catchError { sh ''' if [ $GIT_BRANCH_LOCAL != "main" ];then - gsutil rm -r $jarPath 2> /dev/null || true + gcloud storage rm --recursive $jarPath 2> /dev/null || true fi ''' } diff --git a/java/bin/start.sh b/java/bin/start.sh index e77904643..9b0d21c43 100755 --- a/java/bin/start.sh +++ b/java/bin/start.sh @@ -58,7 +58,7 @@ if [ -z "$SKIP_BUILD" ]; then #Copy jar file to GCS bucket Staging folder echo_formatted "Copying ${PROJECT_ROOT_DIR}/target/${JAR_FILE} to staging bucket: ${GCS_STAGING_LOCATION}/${JAR_FILE}" - gsutil cp ${PROJECT_ROOT_DIR}/target/${JAR_FILE} ${GCS_STAGING_LOCATION}/${JAR_FILE} + gcloud storage cp ${PROJECT_ROOT_DIR}/target/${JAR_FILE} ${GCS_STAGING_LOCATION}/${JAR_FILE} check_status $? "\n Commands to copy the project jar file to GCS Staging location went fine, thus we are good to go \n" "\n It seems like there is some issue in copying the project jar file to GCS Staging location \n" @@ -122,8 +122,8 @@ if [ -n "${CATALOG}" ]; then echo "Downloading Hbase jar dependency" wget https://repo1.maven.org/maven2/org/apache/hbase/hbase-client/2.4.12/hbase-client-2.4.12.jar wget https://repo1.maven.org/maven2/org/apache/hbase/hbase-shaded-mapreduce/2.4.12/hbase-shaded-mapreduce-2.4.12.jar - gsutil copy hbase-client-2.4.12.jar ${GCS_STAGING_LOCATION}/hbase-client-2.4.12.jar - gsutil copy hbase-shaded-mapreduce-2.4.12.jar ${GCS_STAGING_LOCATION}/hbase-shaded-mapreduce-2.4.12.jar + gcloud storage cp hbase-client-2.4.12.jar ${GCS_STAGING_LOCATION}/hbase-client-2.4.12.jar + gcloud storage cp hbase-shaded-mapreduce-2.4.12.jar ${GCS_STAGING_LOCATION}/hbase-shaded-mapreduce-2.4.12.jar echo "Passing downloaded dependency jars" OPT_JARS="${OPT_JARS},${GCS_STAGING_LOCATION}/hbase-client-2.4.12.jar,${GCS_STAGING_LOCATION}/hbase-shaded-mapreduce-2.4.12.jar" rm hbase-client-2.4.12.jar @@ -186,4 +186,3 @@ check_status $spark_status "\n Spark Command ran successful \n" "\n It seems lik echo "We will love to hear your feedback at: https://forms.gle/XXCJeWeCJJ9fNLQS6" echo "Email us at: dataproc-templates-support-external@googlegroups.com" - diff --git a/java/src/main/java/com/google/cloud/dataproc/templates/dataplex/README.md b/java/src/main/java/com/google/cloud/dataproc/templates/dataplex/README.md index 91c667a86..53fb8c5f1 100644 --- a/java/src/main/java/com/google/cloud/dataproc/templates/dataplex/README.md +++ b/java/src/main/java/com/google/cloud/dataproc/templates/dataplex/README.md @@ -11,8 +11,8 @@ from Cloud Storage and the target BQ table will be overwritten. Download jar and properties file ``` export GCS_STAGING_LOCATION=gs://bucket/path/to/staging/folder/ -gsutil -u cp gs://dataplex-dataproc-templates-artifacts/dataproc-templates-1.0-SNAPSHOT.jar ${GCS_STAGING_LOCATION} -gsutil -u cp gs://dataplex-dataproc-templates-artifacts/log4j-spark-driver-template.properties ${GCS_STAGING_LOCATION} +gcloud storage cp gs://dataplex-dataproc-templates-artifacts/dataproc-templates-1.0-SNAPSHOT.jar ${GCS_STAGING_LOCATION} +gcloud storage cp gs://dataplex-dataproc-templates-artifacts/log4j-spark-driver-template.properties ${GCS_STAGING_LOCATION} ``` Create Dataplex Task diff --git a/java/src/main/java/com/google/cloud/dataproc/templates/general/README.md b/java/src/main/java/com/google/cloud/dataproc/templates/general/README.md index 4d929d51f..c472141e1 100644 --- a/java/src/main/java/com/google/cloud/dataproc/templates/general/README.md +++ b/java/src/main/java/com/google/cloud/dataproc/templates/general/README.md @@ -66,7 +66,7 @@ export SUBNET=projects//regions//subnetworks/ Date: Fri, 2 Jan 2026 14:27:51 +0000 Subject: [PATCH 2/4] chore: Migrate gsutil usage to gcloud storage --- java/.ci/dataproc-serverless-integration-tests.yaml | 4 ++-- notebooks/mssql2bq/mssql-to-bigquery-notebook.ipynb | 8 ++++---- .../mssql2postgresql/mssql-to-postgres-notebook.ipynb | 8 ++++---- notebooks/mysql2spanner/MySqlToSpanner_notebook.ipynb | 6 +++--- notebooks/oracle2bq/OracleToBigQuery_notebook.ipynb | 8 ++++---- notebooks/oracle2postgres/OracleToPostgres_notebook.ipynb | 8 ++++---- notebooks/oracle2spanner/OracleToSpanner_notebook.ipynb | 6 +++--- .../postgresql2bq/postgresql-to-bigquery-notebook.ipynb | 8 ++++---- 8 files changed, 28 insertions(+), 28 deletions(-) diff --git a/java/.ci/dataproc-serverless-integration-tests.yaml b/java/.ci/dataproc-serverless-integration-tests.yaml index d1545b511..37851e125 100644 --- a/java/.ci/dataproc-serverless-integration-tests.yaml +++ b/java/.ci/dataproc-serverless-integration-tests.yaml @@ -49,7 +49,7 @@ steps: cd java mvn install spotless:check echo "copying the build into staging location:" ${_GCS_STAGING_LOCATION} - gsutil cp target/$_JAR_FILE ${_GCS_STAGING_LOCATION}/$_JAR_FILE + gcloud storage cp target/$_JAR_FILE ${_GCS_STAGING_LOCATION}/$_JAR_FILE cat > /tmp/employeecatalog.json << EOF { @@ -64,7 +64,7 @@ steps: } EOF cat /tmp/employeecatalog.json - gsutil cp /tmp/employeecatalog.json gs://dataproc-templates_cloudbuild/integration-testing/bigtable/employeecatalog.json + gcloud storage cp /tmp/employeecatalog.json gs://dataproc-templates_cloudbuild/integration-testing/bigtable/employeecatalog.json - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' diff --git a/notebooks/mssql2bq/mssql-to-bigquery-notebook.ipynb b/notebooks/mssql2bq/mssql-to-bigquery-notebook.ipynb index 0720ab8b3..6cea16e93 100644 --- a/notebooks/mssql2bq/mssql-to-bigquery-notebook.ipynb +++ b/notebooks/mssql2bq/mssql-to-bigquery-notebook.ipynb @@ -499,10 +499,10 @@ "metadata": {}, "outputs": [], "source": [ - "! gsutil cp main.py $GCS_STAGING_LOCATION/\n", - "! gsutil cp -r $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/\n", - "! gsutil cp mssql-jdbc-6.4.0.jre8.jar $GCS_STAGING_LOCATION/jars/mssql-jdbc-6.4.0.jre8.jar\n", - "! gsutil cp spark-bigquery-with-dependencies_2.12-0.27.0.jar $GCS_STAGING_LOCATION/jars/spark-bigquery-with-dependencies_2.12-0.27.0.jar" + "! gcloud storage cp main.py $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp --recursive $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp mssql-jdbc-6.4.0.jre8.jar $GCS_STAGING_LOCATION/jars/mssql-jdbc-6.4.0.jre8.jar\n", + "! gcloud storage cp spark-bigquery-with-dependencies_2.12-0.27.0.jar $GCS_STAGING_LOCATION/jars/spark-bigquery-with-dependencies_2.12-0.27.0.jar" ] }, { diff --git a/notebooks/mssql2postgresql/mssql-to-postgres-notebook.ipynb b/notebooks/mssql2postgresql/mssql-to-postgres-notebook.ipynb index 1dce98239..e5c000a4f 100644 --- a/notebooks/mssql2postgresql/mssql-to-postgres-notebook.ipynb +++ b/notebooks/mssql2postgresql/mssql-to-postgres-notebook.ipynb @@ -489,10 +489,10 @@ "metadata": {}, "outputs": [], "source": [ - "! gsutil cp main.py $GCS_STAGING_LOCATION/\n", - "! gsutil cp -r $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/\n", - "! gsutil cp mssql-jdbc-6.4.0.jre8.jar $GCS_STAGING_LOCATION/jars/mssql-jdbc-6.4.0.jre8.jar\n", - "! gsutil cp postgresql-42.2.6.jar $GCS_STAGING_LOCATION/jars/postgresql-42.2.6.jar\n" + "! gcloud storage cp main.py $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp --recursive $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp mssql-jdbc-6.4.0.jre8.jar $GCS_STAGING_LOCATION/jars/mssql-jdbc-6.4.0.jre8.jar\n", + "! gcloud storage cp postgresql-42.2.6.jar $GCS_STAGING_LOCATION/jars/postgresql-42.2.6.jar\n" ] }, { diff --git a/notebooks/mysql2spanner/MySqlToSpanner_notebook.ipynb b/notebooks/mysql2spanner/MySqlToSpanner_notebook.ipynb index 66d397801..21fc16fa8 100644 --- a/notebooks/mysql2spanner/MySqlToSpanner_notebook.ipynb +++ b/notebooks/mysql2spanner/MySqlToSpanner_notebook.ipynb @@ -564,9 +564,9 @@ }, "outputs": [], "source": [ - "!gsutil cp target/$JAR_FILE $GCS_STAGING_LOCATION/$JAR_FILE\n", - "!gsutil cp $LOG4J_PROPERTIES_PATH/$LOG4J_PROPERTIES $GCS_STAGING_LOCATION/$LOG4J_PROPERTIES\n", - "!gsutil cp mysql-connector-java-8.0.29/mysql-connector-java-8.0.29.jar $GCS_STAGING_LOCATION/jars/mysql-connector-java-8.0.29.jar" + "!gcloud storage cp target/$JAR_FILE $GCS_STAGING_LOCATION/$JAR_FILE\n", + "!gcloud storage cp $LOG4J_PROPERTIES_PATH/$LOG4J_PROPERTIES $GCS_STAGING_LOCATION/$LOG4J_PROPERTIES\n", + "!gcloud storage cp mysql-connector-java-8.0.29/mysql-connector-java-8.0.29.jar $GCS_STAGING_LOCATION/jars/mysql-connector-java-8.0.29.jar" ] }, { diff --git a/notebooks/oracle2bq/OracleToBigQuery_notebook.ipynb b/notebooks/oracle2bq/OracleToBigQuery_notebook.ipynb index 4027bbe56..ff5dedab8 100644 --- a/notebooks/oracle2bq/OracleToBigQuery_notebook.ipynb +++ b/notebooks/oracle2bq/OracleToBigQuery_notebook.ipynb @@ -519,10 +519,10 @@ }, "outputs": [], "source": [ - "! gsutil cp main.py $GCS_STAGING_LOCATION/dependencies/\n", - "! gsutil cp -r $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/dependencies/\n", - "! gsutil cp spark-bigquery-with-dependencies_2.12-0.27.0.jar $GCS_STAGING_LOCATION/jars/spark-bigquery-with-dependencies_2.12-0.27.0.jar\n", - "! gsutil cp $OJDBC_JAR $GCS_STAGING_LOCATION/jars/$OJDBC_JAR" + "! gcloud storage cp main.py $GCS_STAGING_LOCATION/dependencies/\n", + "! gcloud storage cp --recursive $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/dependencies/\n", + "! gcloud storage cp spark-bigquery-with-dependencies_2.12-0.27.0.jar $GCS_STAGING_LOCATION/jars/spark-bigquery-with-dependencies_2.12-0.27.0.jar\n", + "! gcloud storage cp $OJDBC_JAR $GCS_STAGING_LOCATION/jars/$OJDBC_JAR" ] }, { diff --git a/notebooks/oracle2postgres/OracleToPostgres_notebook.ipynb b/notebooks/oracle2postgres/OracleToPostgres_notebook.ipynb index 5241bdf4c..2178884c0 100644 --- a/notebooks/oracle2postgres/OracleToPostgres_notebook.ipynb +++ b/notebooks/oracle2postgres/OracleToPostgres_notebook.ipynb @@ -607,10 +607,10 @@ }, "outputs": [], "source": [ - "! gsutil cp main.py $GCS_STAGING_LOCATION/\n", - "! gsutil cp -r $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/\n", - "! gsutil cp $OJDBC_JAR $GCS_STAGING_LOCATION/jars/$OJDBC_JAR\n", - "! gsutil cp postgresql-42.2.6.jar $GCS_STAGING_LOCATION/jars/postgresql-42.2.6.jar" + "! gcloud storage cp main.py $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp --recursive $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp $OJDBC_JAR $GCS_STAGING_LOCATION/jars/$OJDBC_JAR\n", + "! gcloud storage cp postgresql-42.2.6.jar $GCS_STAGING_LOCATION/jars/postgresql-42.2.6.jar" ] }, { diff --git a/notebooks/oracle2spanner/OracleToSpanner_notebook.ipynb b/notebooks/oracle2spanner/OracleToSpanner_notebook.ipynb index 0fceaaa34..78cf3d988 100644 --- a/notebooks/oracle2spanner/OracleToSpanner_notebook.ipynb +++ b/notebooks/oracle2spanner/OracleToSpanner_notebook.ipynb @@ -544,9 +544,9 @@ }, "outputs": [], "source": [ - "!gsutil cp target/$JAR_FILE $GCS_STAGING_LOCATION/$JAR_FILE\n", - "!gsutil cp $LOG4J_PROPERTIES_PATH/$LOG4J_PROPERTIES $GCS_STAGING_LOCATION/$LOG4J_PROPERTIES\n", - "!gsutil cp $OJDBC_JAR $GCS_STAGING_LOCATION/jars/$OJDBC_JAR" + "!gcloud storage cp target/$JAR_FILE $GCS_STAGING_LOCATION/$JAR_FILE\n", + "!gcloud storage cp $LOG4J_PROPERTIES_PATH/$LOG4J_PROPERTIES $GCS_STAGING_LOCATION/$LOG4J_PROPERTIES\n", + "!gcloud storage cp $OJDBC_JAR $GCS_STAGING_LOCATION/jars/$OJDBC_JAR" ] }, { diff --git a/notebooks/postgresql2bq/postgresql-to-bigquery-notebook.ipynb b/notebooks/postgresql2bq/postgresql-to-bigquery-notebook.ipynb index e4ceb4708..a1cb3f981 100644 --- a/notebooks/postgresql2bq/postgresql-to-bigquery-notebook.ipynb +++ b/notebooks/postgresql2bq/postgresql-to-bigquery-notebook.ipynb @@ -659,10 +659,10 @@ }, "outputs": [], "source": [ - "! gsutil cp main.py $GCS_STAGING_LOCATION/\n", - "! gsutil cp -r $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/\n", - "! gsutil cp postgresql-42.2.6.jar $GCS_STAGING_LOCATION/jars/postgresql-42.2.6.jar\n", - "! gsutil cp spark-bigquery-with-dependencies_2.12-0.27.0.jar $GCS_STAGING_LOCATION/jars/spark-bigquery-with-dependencies_2.12-0.27.0.jar" + "! gcloud storage cp main.py $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp --recursive $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp postgresql-42.2.6.jar $GCS_STAGING_LOCATION/jars/postgresql-42.2.6.jar\n", + "! gcloud storage cp spark-bigquery-with-dependencies_2.12-0.27.0.jar $GCS_STAGING_LOCATION/jars/spark-bigquery-with-dependencies_2.12-0.27.0.jar" ] }, { From f5828de6dc8979e814c5b81f43bc784d571bba02 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 2 Jan 2026 20:42:46 +0530 Subject: [PATCH 3/4] Update README.md --- .../com/google/cloud/dataproc/templates/dataplex/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/java/src/main/java/com/google/cloud/dataproc/templates/dataplex/README.md b/java/src/main/java/com/google/cloud/dataproc/templates/dataplex/README.md index 53fb8c5f1..27af6d1f8 100644 --- a/java/src/main/java/com/google/cloud/dataproc/templates/dataplex/README.md +++ b/java/src/main/java/com/google/cloud/dataproc/templates/dataplex/README.md @@ -11,8 +11,8 @@ from Cloud Storage and the target BQ table will be overwritten. Download jar and properties file ``` export GCS_STAGING_LOCATION=gs://bucket/path/to/staging/folder/ -gcloud storage cp gs://dataplex-dataproc-templates-artifacts/dataproc-templates-1.0-SNAPSHOT.jar ${GCS_STAGING_LOCATION} -gcloud storage cp gs://dataplex-dataproc-templates-artifacts/log4j-spark-driver-template.properties ${GCS_STAGING_LOCATION} +gcloud storage cp gs://dataplex-dataproc-templates-artifacts/dataproc-templates-1.0-SNAPSHOT.jar ${GCS_STAGING_LOCATION} --billing-project= +gcloud storage cp gs://dataplex-dataproc-templates-artifacts/log4j-spark-driver-template.properties ${GCS_STAGING_LOCATION} --billing-project= ``` Create Dataplex Task @@ -101,4 +101,4 @@ FROM __table__ WHERE id > 100 -``` \ No newline at end of file +``` From f43aef055ee3d24e48015b56118dc00d43787778 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 2 Jan 2026 15:17:00 +0000 Subject: [PATCH 4/4] updates --- notebooks/hive2bq/HiveToBigquery_notebook.ipynb | 10 +++++----- python/.ci/Jenkinsfile | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/notebooks/hive2bq/HiveToBigquery_notebook.ipynb b/notebooks/hive2bq/HiveToBigquery_notebook.ipynb index c84d5c33d..9ee87244a 100644 --- a/notebooks/hive2bq/HiveToBigquery_notebook.ipynb +++ b/notebooks/hive2bq/HiveToBigquery_notebook.ipynb @@ -346,8 +346,8 @@ }, "outputs": [], "source": [ - "!gsutil cp main.py $GCS_STAGING_LOCATION/\n", - "!gsutil cp $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/dist/" + "! gcloud storage cp main.py $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/dist/" ] }, { @@ -474,7 +474,7 @@ }, "outputs": [], "source": [ - "result=subprocess.run(f\"gsutil ls {DDL_INPUT_PATH}/{INPUT_HIVE_DATABASE} | sed '$!d'\", capture_output=True, shell=True, encoding=\"utf-8\")\n", + "result=subprocess.run(f\"gcloud storage ls {DDL_INPUT_PATH}/{INPUT_HIVE_DATABASE} | sed '$!d'\", capture_output=True, shell=True, encoding=\"utf-8\")\n", "latest_ddl_path=str(result.stdout)[:-1]\n", "print(latest_ddl_path)" ] @@ -500,7 +500,7 @@ }, "outputs": [], "source": [ - "!gsutil cp ../notebooks/util/global_typeconvert.config.yaml \"$latest_ddl_path\"" + "! gcloud storage cp ../notebooks/util/global_typeconvert.config.yaml \"$latest_ddl_path\"" ] }, { @@ -604,7 +604,7 @@ "outputs": [], "source": [ "from google.cloud import storage\n", - "result=subprocess.run(f\"gsutil ls {gcs_output_path} \", capture_output=True, shell=True, encoding=\"utf-8\")\n", + "result=subprocess.run(f\"gcloud storage ls {gcs_output_path} \", capture_output=True, shell=True, encoding=\"utf-8\")\n", "translated_files=result.stdout.split(\"\\n\")\n", "all_ddls=\"\"\n", "client = storage.Client()\n", diff --git a/python/.ci/Jenkinsfile b/python/.ci/Jenkinsfile index 788913b0b..f9e2394ef 100644 --- a/python/.ci/Jenkinsfile +++ b/python/.ci/Jenkinsfile @@ -704,7 +704,7 @@ pipeline { // source env/bin/activate - // gsutil cp gs://python-dataproc-templates/surjitsh/hbase-site.xml . + // gcloud storage cp gs://python-dataproc-templates/surjitsh/hbase-site.xml . // export GCS_STAGING_LOCATION=gs://python-dataproc-templates-temp // export CATALOG='{"table":{"namespace":"default","name":"my_table"},"rowkey":"key","columns":{"key":{"cf":"rowkey","col":"key","type":"string"},"name":{"cf":"cf","col":"name","type":"string"}}}' @@ -800,7 +800,7 @@ EOF cat /tmp/cities.json - gsutil cp /tmp/cities.json gs://dataproc-templates/conf/ + gcloud storage cp /tmp/cities.json gs://dataproc-templates/conf/ export GCS_STAGING_LOCATION="gs://python-dataproc-templates-temp" export SKIP_BUILD=true export JARS="gs://spark-lib/bigtable/spark-bigtable_2.12-0.1.0.jar" @@ -834,7 +834,7 @@ EOF source env/bin/activate - gsutil cp gs://dataproc-templates/conf/elastic.jks /tmp/ + gcloud storage cp gs://dataproc-templates/conf/elastic.jks /tmp/ export GCS_STAGING_LOCATION="gs://python-dataproc-templates-temp" export SKIP_BUILD=true