#Walmart de México
# Autor : Krissel Mejia <Bluetab>
# UserID : vn52l97
# Creación : 04-Diciembre-2023
# ver 1.0 : 04-Diciembre-2023
#------------------------------------------------------------#
# Propósito : Extración delta de 30 días de la tabla de IDC_HEADER_TENDER de BCT
para un intervalo especifico.
# Proyecto: Datamesh
#------------------------------------------------------------#
---
name: "bct-we-inc-mx_bct_hs-idc_hdr_tndr-hs"
version: "1.0"
cluster_name: "bct-we-inc-mx-bct-hs-master-extraction"
cluster_profile: "medium"
data_group: "core_hs"
path: "incremental/mx_bct_hs/we/idc_hdr_tndr/"
properties_file: "inc_mx_bct_hs_idc_hdr_tndr_properties.yaml"
# Workflow Tasks
tasks:
- name: "start"
type: "START"
description: ""
long_name: "start"
curr_id: 1
properties: {}
- name: "bct_get_days"
type: "SCRIPT"
description: "execute dtmsh_get_bct_dates_idc_hdr_tndr to get extraction dates
for bct incremental process"
long_name:
"$config_bucket/scripts/dtmsh_general_scripts/bct/idc_hdr_tndr/dtmsh_get_bct_dates_
idc_hdr_tndr.sh"
curr_id: 2
parents:
- "start"
properties:
params:
"target_schema=$target_schema,target_table=$target_table,process_type=$process_type
,division_code=$division_code"
script_file: "dtmsh_get_bct_dates_idc_hdr_tndr.sh"
script_location:
"$config_bucket/scripts/dtmsh_general_scripts/bct/idc_hdr_tndr/"
- name: "bct_mssql_to_lakestorage"
type: "SQOOP"
description: "Sqoop Load to GCP"
long_name: "01_inc_mx_bct_hs_idc_hdr_tndr_sql_to_gcs"
curr_id: 3
parents:
- "bct_get_days"
properties_file: "01_inc_mx_bct_hs_idc_hdr_tndr_sql_to_gcs.yaml"
- name: "bct_lakestorage_to_raw"
type: "HIVE"
description: "Hive Yaml Raw Load"
long_name: "02_inc_mx_bct_hs_idc_hdr_tndr_gcs_to_raw"
curr_id: 4
parents:
- "bct_mssql_to_lakestorage"
properties_file: "02_inc_mx_bct_hs_idc_hdr_tndr_gcs_to_raw.yaml"
- name: "bct_raw_to_ctg"
type: "HIVE"
description: "Hive Yaml Ctg Load"
long_name: "03_inc_mx_bct_hs_idc_hdr_tndr_raw_to_ctg"
curr_id: 5
parents:
- "bct_lakestorage_to_raw"
properties_file: "03_inc_mx_bct_hs_idc_hdr_tndr_raw_to_ctg.yaml"
- name: "bct_archive"
type: "SCRIPT"
description: "execute dtmsh_gcp_utils_idc_hdr_tndr to copy data to archive
folder."
long_name:
"$config_bucket/scripts/dtmsh_general_scripts/bct/idc_hdr_tndr/dtmsh_gcp_utils_idc_
hdr_tndr.sh"
curr_id: 6
parents:
- "bct_raw_to_ctg"
properties:
params:
"operation_type=6,archive_flag=$archive_flag,delete_flag=$delete_flag,source_bucket
=$raw_bucket,target_bucket=$archive_bucket,source_path_object=landing/$load_type/
$target_schema/$target_table/$division_code/$end_date,target_path_object=landing/
archive/$load_type/$target_schema/$target_table/$division_code/"
script_file: "dtmsh_gcp_utils_idc_hdr_tndr.sh"
script_location:
"$config_bucket/scripts/dtmsh_general_scripts/bct/idc_hdr_tndr/"
- name: "end"
type: "END"
description: "end"
long_name: "end"
curr_id: 7
parents:
- "bct_archive"
properties:
done_file_path: "$done_bucket/$target_schema/$target_table/$geo_region_cd"
done_file_name: "$target_schema_$target_table_YYYYMMddHHmmss.done"
