From c415388a70192cbcae827ef3b0427893493394a6 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 2 Jan 2026 14:27:51 +0000 Subject: [PATCH 1/2] chore: Migrate gsutil usage to gcloud storage --- java/.ci/dataproc-serverless-integration-tests.yaml | 4 ++-- notebooks/mssql2bq/mssql-to-bigquery-notebook.ipynb | 8 ++++---- .../mssql2postgresql/mssql-to-postgres-notebook.ipynb | 8 ++++---- notebooks/mysql2spanner/MySqlToSpanner_notebook.ipynb | 6 +++--- notebooks/oracle2bq/OracleToBigQuery_notebook.ipynb | 8 ++++---- notebooks/oracle2postgres/OracleToPostgres_notebook.ipynb | 8 ++++---- notebooks/oracle2spanner/OracleToSpanner_notebook.ipynb | 6 +++--- .../postgresql2bq/postgresql-to-bigquery-notebook.ipynb | 8 ++++---- 8 files changed, 28 insertions(+), 28 deletions(-) diff --git a/java/.ci/dataproc-serverless-integration-tests.yaml b/java/.ci/dataproc-serverless-integration-tests.yaml index d1545b511..37851e125 100644 --- a/java/.ci/dataproc-serverless-integration-tests.yaml +++ b/java/.ci/dataproc-serverless-integration-tests.yaml @@ -49,7 +49,7 @@ steps: cd java mvn install spotless:check echo "copying the build into staging location:" ${_GCS_STAGING_LOCATION} - gsutil cp target/$_JAR_FILE ${_GCS_STAGING_LOCATION}/$_JAR_FILE + gcloud storage cp target/$_JAR_FILE ${_GCS_STAGING_LOCATION}/$_JAR_FILE cat > /tmp/employeecatalog.json << EOF { @@ -64,7 +64,7 @@ steps: } EOF cat /tmp/employeecatalog.json - gsutil cp /tmp/employeecatalog.json gs://dataproc-templates_cloudbuild/integration-testing/bigtable/employeecatalog.json + gcloud storage cp /tmp/employeecatalog.json gs://dataproc-templates_cloudbuild/integration-testing/bigtable/employeecatalog.json - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' diff --git a/notebooks/mssql2bq/mssql-to-bigquery-notebook.ipynb b/notebooks/mssql2bq/mssql-to-bigquery-notebook.ipynb index 0720ab8b3..6cea16e93 100644 --- a/notebooks/mssql2bq/mssql-to-bigquery-notebook.ipynb +++ b/notebooks/mssql2bq/mssql-to-bigquery-notebook.ipynb @@ -499,10 +499,10 @@ "metadata": {}, "outputs": [], "source": [ - "! gsutil cp main.py $GCS_STAGING_LOCATION/\n", - "! gsutil cp -r $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/\n", - "! gsutil cp mssql-jdbc-6.4.0.jre8.jar $GCS_STAGING_LOCATION/jars/mssql-jdbc-6.4.0.jre8.jar\n", - "! gsutil cp spark-bigquery-with-dependencies_2.12-0.27.0.jar $GCS_STAGING_LOCATION/jars/spark-bigquery-with-dependencies_2.12-0.27.0.jar" + "! gcloud storage cp main.py $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp --recursive $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp mssql-jdbc-6.4.0.jre8.jar $GCS_STAGING_LOCATION/jars/mssql-jdbc-6.4.0.jre8.jar\n", + "! gcloud storage cp spark-bigquery-with-dependencies_2.12-0.27.0.jar $GCS_STAGING_LOCATION/jars/spark-bigquery-with-dependencies_2.12-0.27.0.jar" ] }, { diff --git a/notebooks/mssql2postgresql/mssql-to-postgres-notebook.ipynb b/notebooks/mssql2postgresql/mssql-to-postgres-notebook.ipynb index 1dce98239..e5c000a4f 100644 --- a/notebooks/mssql2postgresql/mssql-to-postgres-notebook.ipynb +++ b/notebooks/mssql2postgresql/mssql-to-postgres-notebook.ipynb @@ -489,10 +489,10 @@ "metadata": {}, "outputs": [], "source": [ - "! gsutil cp main.py $GCS_STAGING_LOCATION/\n", - "! gsutil cp -r $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/\n", - "! gsutil cp mssql-jdbc-6.4.0.jre8.jar $GCS_STAGING_LOCATION/jars/mssql-jdbc-6.4.0.jre8.jar\n", - "! gsutil cp postgresql-42.2.6.jar $GCS_STAGING_LOCATION/jars/postgresql-42.2.6.jar\n" + "! gcloud storage cp main.py $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp --recursive $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp mssql-jdbc-6.4.0.jre8.jar $GCS_STAGING_LOCATION/jars/mssql-jdbc-6.4.0.jre8.jar\n", + "! gcloud storage cp postgresql-42.2.6.jar $GCS_STAGING_LOCATION/jars/postgresql-42.2.6.jar\n" ] }, { diff --git a/notebooks/mysql2spanner/MySqlToSpanner_notebook.ipynb b/notebooks/mysql2spanner/MySqlToSpanner_notebook.ipynb index 66d397801..21fc16fa8 100644 --- a/notebooks/mysql2spanner/MySqlToSpanner_notebook.ipynb +++ b/notebooks/mysql2spanner/MySqlToSpanner_notebook.ipynb @@ -564,9 +564,9 @@ }, "outputs": [], "source": [ - "!gsutil cp target/$JAR_FILE $GCS_STAGING_LOCATION/$JAR_FILE\n", - "!gsutil cp $LOG4J_PROPERTIES_PATH/$LOG4J_PROPERTIES $GCS_STAGING_LOCATION/$LOG4J_PROPERTIES\n", - "!gsutil cp mysql-connector-java-8.0.29/mysql-connector-java-8.0.29.jar $GCS_STAGING_LOCATION/jars/mysql-connector-java-8.0.29.jar" + "!gcloud storage cp target/$JAR_FILE $GCS_STAGING_LOCATION/$JAR_FILE\n", + "!gcloud storage cp $LOG4J_PROPERTIES_PATH/$LOG4J_PROPERTIES $GCS_STAGING_LOCATION/$LOG4J_PROPERTIES\n", + "!gcloud storage cp mysql-connector-java-8.0.29/mysql-connector-java-8.0.29.jar $GCS_STAGING_LOCATION/jars/mysql-connector-java-8.0.29.jar" ] }, { diff --git a/notebooks/oracle2bq/OracleToBigQuery_notebook.ipynb b/notebooks/oracle2bq/OracleToBigQuery_notebook.ipynb index 4027bbe56..ff5dedab8 100644 --- a/notebooks/oracle2bq/OracleToBigQuery_notebook.ipynb +++ b/notebooks/oracle2bq/OracleToBigQuery_notebook.ipynb @@ -519,10 +519,10 @@ }, "outputs": [], "source": [ - "! gsutil cp main.py $GCS_STAGING_LOCATION/dependencies/\n", - "! gsutil cp -r $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/dependencies/\n", - "! gsutil cp spark-bigquery-with-dependencies_2.12-0.27.0.jar $GCS_STAGING_LOCATION/jars/spark-bigquery-with-dependencies_2.12-0.27.0.jar\n", - "! gsutil cp $OJDBC_JAR $GCS_STAGING_LOCATION/jars/$OJDBC_JAR" + "! gcloud storage cp main.py $GCS_STAGING_LOCATION/dependencies/\n", + "! gcloud storage cp --recursive $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/dependencies/\n", + "! gcloud storage cp spark-bigquery-with-dependencies_2.12-0.27.0.jar $GCS_STAGING_LOCATION/jars/spark-bigquery-with-dependencies_2.12-0.27.0.jar\n", + "! gcloud storage cp $OJDBC_JAR $GCS_STAGING_LOCATION/jars/$OJDBC_JAR" ] }, { diff --git a/notebooks/oracle2postgres/OracleToPostgres_notebook.ipynb b/notebooks/oracle2postgres/OracleToPostgres_notebook.ipynb index 5241bdf4c..2178884c0 100644 --- a/notebooks/oracle2postgres/OracleToPostgres_notebook.ipynb +++ b/notebooks/oracle2postgres/OracleToPostgres_notebook.ipynb @@ -607,10 +607,10 @@ }, "outputs": [], "source": [ - "! gsutil cp main.py $GCS_STAGING_LOCATION/\n", - "! gsutil cp -r $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/\n", - "! gsutil cp $OJDBC_JAR $GCS_STAGING_LOCATION/jars/$OJDBC_JAR\n", - "! gsutil cp postgresql-42.2.6.jar $GCS_STAGING_LOCATION/jars/postgresql-42.2.6.jar" + "! gcloud storage cp main.py $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp --recursive $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp $OJDBC_JAR $GCS_STAGING_LOCATION/jars/$OJDBC_JAR\n", + "! gcloud storage cp postgresql-42.2.6.jar $GCS_STAGING_LOCATION/jars/postgresql-42.2.6.jar" ] }, { diff --git a/notebooks/oracle2spanner/OracleToSpanner_notebook.ipynb b/notebooks/oracle2spanner/OracleToSpanner_notebook.ipynb index 0fceaaa34..78cf3d988 100644 --- a/notebooks/oracle2spanner/OracleToSpanner_notebook.ipynb +++ b/notebooks/oracle2spanner/OracleToSpanner_notebook.ipynb @@ -544,9 +544,9 @@ }, "outputs": [], "source": [ - "!gsutil cp target/$JAR_FILE $GCS_STAGING_LOCATION/$JAR_FILE\n", - "!gsutil cp $LOG4J_PROPERTIES_PATH/$LOG4J_PROPERTIES $GCS_STAGING_LOCATION/$LOG4J_PROPERTIES\n", - "!gsutil cp $OJDBC_JAR $GCS_STAGING_LOCATION/jars/$OJDBC_JAR" + "!gcloud storage cp target/$JAR_FILE $GCS_STAGING_LOCATION/$JAR_FILE\n", + "!gcloud storage cp $LOG4J_PROPERTIES_PATH/$LOG4J_PROPERTIES $GCS_STAGING_LOCATION/$LOG4J_PROPERTIES\n", + "!gcloud storage cp $OJDBC_JAR $GCS_STAGING_LOCATION/jars/$OJDBC_JAR" ] }, { diff --git a/notebooks/postgresql2bq/postgresql-to-bigquery-notebook.ipynb b/notebooks/postgresql2bq/postgresql-to-bigquery-notebook.ipynb index e4ceb4708..a1cb3f981 100644 --- a/notebooks/postgresql2bq/postgresql-to-bigquery-notebook.ipynb +++ b/notebooks/postgresql2bq/postgresql-to-bigquery-notebook.ipynb @@ -659,10 +659,10 @@ }, "outputs": [], "source": [ - "! gsutil cp main.py $GCS_STAGING_LOCATION/\n", - "! gsutil cp -r $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/\n", - "! gsutil cp postgresql-42.2.6.jar $GCS_STAGING_LOCATION/jars/postgresql-42.2.6.jar\n", - "! gsutil cp spark-bigquery-with-dependencies_2.12-0.27.0.jar $GCS_STAGING_LOCATION/jars/spark-bigquery-with-dependencies_2.12-0.27.0.jar" + "! gcloud storage cp main.py $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp --recursive $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp postgresql-42.2.6.jar $GCS_STAGING_LOCATION/jars/postgresql-42.2.6.jar\n", + "! gcloud storage cp spark-bigquery-with-dependencies_2.12-0.27.0.jar $GCS_STAGING_LOCATION/jars/spark-bigquery-with-dependencies_2.12-0.27.0.jar" ] }, { From f43aef055ee3d24e48015b56118dc00d43787778 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 2 Jan 2026 15:17:00 +0000 Subject: [PATCH 2/2] updates --- notebooks/hive2bq/HiveToBigquery_notebook.ipynb | 10 +++++----- python/.ci/Jenkinsfile | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/notebooks/hive2bq/HiveToBigquery_notebook.ipynb b/notebooks/hive2bq/HiveToBigquery_notebook.ipynb index c84d5c33d..9ee87244a 100644 --- a/notebooks/hive2bq/HiveToBigquery_notebook.ipynb +++ b/notebooks/hive2bq/HiveToBigquery_notebook.ipynb @@ -346,8 +346,8 @@ }, "outputs": [], "source": [ - "!gsutil cp main.py $GCS_STAGING_LOCATION/\n", - "!gsutil cp $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/dist/" + "! gcloud storage cp main.py $GCS_STAGING_LOCATION/\n", + "! gcloud storage cp $PACKAGE_EGG_FILE $GCS_STAGING_LOCATION/dist/" ] }, { @@ -474,7 +474,7 @@ }, "outputs": [], "source": [ - "result=subprocess.run(f\"gsutil ls {DDL_INPUT_PATH}/{INPUT_HIVE_DATABASE} | sed '$!d'\", capture_output=True, shell=True, encoding=\"utf-8\")\n", + "result=subprocess.run(f\"gcloud storage ls {DDL_INPUT_PATH}/{INPUT_HIVE_DATABASE} | sed '$!d'\", capture_output=True, shell=True, encoding=\"utf-8\")\n", "latest_ddl_path=str(result.stdout)[:-1]\n", "print(latest_ddl_path)" ] @@ -500,7 +500,7 @@ }, "outputs": [], "source": [ - "!gsutil cp ../notebooks/util/global_typeconvert.config.yaml \"$latest_ddl_path\"" + "! gcloud storage cp ../notebooks/util/global_typeconvert.config.yaml \"$latest_ddl_path\"" ] }, { @@ -604,7 +604,7 @@ "outputs": [], "source": [ "from google.cloud import storage\n", - "result=subprocess.run(f\"gsutil ls {gcs_output_path} \", capture_output=True, shell=True, encoding=\"utf-8\")\n", + "result=subprocess.run(f\"gcloud storage ls {gcs_output_path} \", capture_output=True, shell=True, encoding=\"utf-8\")\n", "translated_files=result.stdout.split(\"\\n\")\n", "all_ddls=\"\"\n", "client = storage.Client()\n", diff --git a/python/.ci/Jenkinsfile b/python/.ci/Jenkinsfile index 788913b0b..f9e2394ef 100644 --- a/python/.ci/Jenkinsfile +++ b/python/.ci/Jenkinsfile @@ -704,7 +704,7 @@ pipeline { // source env/bin/activate - // gsutil cp gs://python-dataproc-templates/surjitsh/hbase-site.xml . + // gcloud storage cp gs://python-dataproc-templates/surjitsh/hbase-site.xml . // export GCS_STAGING_LOCATION=gs://python-dataproc-templates-temp // export CATALOG='{"table":{"namespace":"default","name":"my_table"},"rowkey":"key","columns":{"key":{"cf":"rowkey","col":"key","type":"string"},"name":{"cf":"cf","col":"name","type":"string"}}}' @@ -800,7 +800,7 @@ EOF cat /tmp/cities.json - gsutil cp /tmp/cities.json gs://dataproc-templates/conf/ + gcloud storage cp /tmp/cities.json gs://dataproc-templates/conf/ export GCS_STAGING_LOCATION="gs://python-dataproc-templates-temp" export SKIP_BUILD=true export JARS="gs://spark-lib/bigtable/spark-bigtable_2.12-0.1.0.jar" @@ -834,7 +834,7 @@ EOF source env/bin/activate - gsutil cp gs://dataproc-templates/conf/elastic.jks /tmp/ + gcloud storage cp gs://dataproc-templates/conf/elastic.jks /tmp/ export GCS_STAGING_LOCATION="gs://python-dataproc-templates-temp" export SKIP_BUILD=true