airflow从datahub获取数据到mysql数据库

最新推荐文章于 2025-06-15 09:00:00 发布

他强任他强03

最新推荐文章于 2025-06-15 09:00:00 发布

阅读量628

点赞数

CC 4.0 BY-SA版权

分类专栏： python 文章标签：数据库 mysql pandas etl

本文链接：https://blog.csdn.net/nansen0324/article/details/130304621

使用airflow从datahub获取数据到DB，修改数据，打包csv档发送minio，task执行成功邮件通知复写等

import smtplib
import pendulum
from email.mime.text import MIMEText
from email.header import Header
from airflow import DAG
from airflow.utils.dates import days_ago
from datahub.operators.datahub_to_rdb import DatahubToRDBOperator
from datetime import datetime, timedelta
from airflow.operators.python import PythonOperator
from airflow.operators.dummy import DummyOperator
from airflow.providers.mysql.operators.mysql import MySqlOperator
import pandas as pd
import boto3
from airflow.hooks.mysql_hook import MySqlHook
import tempfile
from airflow.exceptions import AirflowException
import math


def  readdata():
    # 创建mysql引擎
    mysql_hook=MySqlHook(mysql_conn_id="数据库连接id")
    sqlalchemy_engine=mysql_hook.get_sqlalchemy_engine();
    try:
      df=pd.read_sql("SELECT *   FROM  employeeinfo  ",sqlalchemy_engine)
      print(df.head())
      print('--------------')
      print(df.info())
      #生成test.csv档，并设置索引为false
      df.to_csv("test.csv",encoding=