export REGION=us-central1 export ZONE=us-central1-f export PROJECT='sam-gcp-learn' export COMPOSER_ENVIRONMENT='sam-environment' gcloud config set compute/zone $ZONE gcloud config set project $PROJECT
gcloud composer environments create $COMPOSER_ENVIRONMENT
--location us-central1
--project $PROJECT
gcloud composer environments storage dags import
--environment $COMPOSER_ENVIRONMENT
--location us-central1
--source quickstart.py
gcloud composer environments storage dags import
--environment $COMPOSER_ENVIRONMENT
--location us-central1
--source GcsToBigQueryTriggered.py
dataset: test table: usa_names
| Column Name | Column Type |
|---|---|
| state | STRING |
| gender | STRING |
| year | STRING |
| name | STRING |
| number | STRING |
| created_date | STRING |
| filename | STRING |
| load_dt | DATE |
gsutil mb gs://sam-gcp-input gsutil mb gs://sam-gcp-output gsutil mb gs://sam-gcp-temp
gcloud composer environments run $COMPOSER_ENVIRONMENT variables --location us-central1
-- --set gcp_project $PROJECT
gcloud composer environments run $COMPOSER_ENVIRONMENT variables --location us-central1
-- --set gcp_temp_location gs://sam-gcp-temp/tmp
gcloud composer environments run $COMPOSER_ENVIRONMENT variables --location us-central1
-- --set gcs_completion_bucket sam-gcp-output
gcloud composer environments run $COMPOSER_ENVIRONMENT variables --location us-central1
-- --set input_field_names state,gender,year,name,number,created_date
gcloud composer environments run $COMPOSER_ENVIRONMENT variables --location us-central1
-- --set bq_output_table test.usa_names
gcloud composer environments run $COMPOSER_ENVIRONMENT variables --location us-central1
-- --set email some-email@mycompany.com
create dataflow folder inside dags folder and upload the file
https://cloud.google.com/composer/docs/how-to/using/triggering-with-gcf
- Ensure that the DAG_NAME property is set to GcsToBigQueryTriggered i.e. The DAG name defined in simple_load_dag .py.
gcloud iam service-accounts add-iam-policy-binding
$PROJECT@appspot.gserviceaccount.com
--member=serviceAccount:$PROJECT@appspot.gserviceaccount.com
--role=roles/iam.serviceAccountTokenCreator
gcloud functions deploy trigger_dag --runtime python37 --trigger-bucket sam-gcp-input
- Google Cloud Documentation