纯文字网页数据的批量下载--以NOAA为例

import requests
import os


def download():
    """下载网址为： https://gml.noaa.gov/aftp/data/radiation/surfrad/aod/fpk"""
    # 前闭后开，后面 + 1
    for year in range(1997, 2021 + 1):
        for month in range(1, 12 + 1):
            for day in range(1, 31 + 1):

                # 文件名拼接， 样例为：fpk_19970406.aod
                # y = str(year)[2:]
                if month < 10:
                    m = '0' + str(month)
                else:
                    m = str(month)
                if day < 10:
                    d = '0' + str(day)
                else:
                    d = str(day)
                time = str(year) + str(m) + str(d)
                file_name = "fpk_" + time + ".aod"

                # 文件存储地址
                file_path = os.path.join("./data/aod/fpk/", str(year) + '/', file_name)

                # 网页请求地址
                html_path = os.path.join("https://gml.noaa.gov/aftp/data/radiation/surfrad/aod/fpk/", str(year) + '/',
                                         file_name)
                print(file_path)
                print(html_path)

                response = requests.get(html_path)
                print(response.status_code)

                # 有些数据缺失，故对响应状态进行判断
                if response.status_code != 404:
                    (filepath, tempfilename) = os.path.split(file_path)
                    if not os.path.exists(filepath):
                        os.makedirs(filepath)
                    with open(file_path, "w") as file:
                        file_data = response.content.decode('gbk')
                        # print(file_data)
                        if file_data:
                            file.write(file_data)


def main():
    download()


if __name__ == '__main__':
    main()

import os.path
from ftplib import FTP
from pathlib import Path


def main():
    for year in range(1997, 2021 + 1):
        html_str = os.path.join("/data/radiation/surfrad/aod/gwn/", str(year))
        file_path = os.path.join("D:/js/python/code/data/aod/gwn/", str(year))
        print(html_str)
        print(file_path)

        if not os.path.exists(file_path):
            os.makedirs(file_path)

        ftp = FTP("ftp.gml.noaa.gov")
        ftp.login()
        # https://gml.noaa.gov/aftp/data/radiation/surfrad/aod/gwn/
        # ftp.cwd('/data/radiation/surfrad/aod/gwn/')
        ftp.cwd(html_str)

        # Get all files
        files = ftp.nlst()

        # Download all the files to C:\Temp
        for file in files:
            print("Downloading..." + file)
            # ftp.retrbinary(f'RETR {file}', open(str(Path(r'C:\Users\z6q6k6\Desktop\aaa') / file), 'wb').write)
            ftp.retrbinary(f'RETR {file}', open(str(Path(file_path) / file), 'wb').write)
        ftp.close()


if __name__ == '__main__':
    main()