python sftp.get 同步远程文件夹实践二-CSDN博客

本文介绍了一个使用Python编写的远程文件同步工具，通过Paramiko库实现SSH连接，支持目录递归复制，适用于批量传输场景。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

主要用途，从本机获取远端机器的文件夹

#!/usr/bin/python   

import pexpect

import paramiko  

import os

import sys

import time

import multiprocessing

import datetime

import crash_on_ipy

from stat import S_ISDIR

ip_list = []
#room_id = sys.argv[1]

cur_time = datetime.datetime.now()

class get_thread():

    def __init__(self,host_ip='ip'):

        self.host_ip='ip'

        self.scp = paramiko.SSHClient()

        self.scp.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        self.scp = paramiko.Transport((host_ip, 22))

        self.scp.connect(username='tomcat', password='xx')

        self.sftp = paramiko.SFTPClient.from_transport(self.scp)

    def _walk_remote(self, dirpath):

        dirnames = []

        filenames = []
 
        for fd in self.sftp.listdir_attr(dirpath):

            if S_ISDIR(fd.st_mode):

                dirnames.append(fd.filename)

            else:

                filenames.append(fd.filename)

        yield dirpath, dirnames, filenames
 
        for dirname in dirnames:

            new_dirpath = os.path.join(dirpath, dirname)

            # yield from self._walk_remote(new_dirpath)

            for walk in self._walk_remote(new_dirpath):

                yield walk
 
    def getall(self,local,remote):

        st_mode = self.sftp.stat(remote).st_mode

        if not S_ISDIR(st_mode):

            filename = os.path.basename(remote)

            self.sftp.get(remote, os.path.join(local, filename))

        else:

            parent, child = os.path.split(remote)
 
            for dirpath, dirnames, filenames in self._walk_remote(remote):

                dirpath = dirpath.replace(parent, '.')

                for dirname in dirnames:

                        try:

                           os.makedirs(os.path.join(local, dirpath, dirname))

                                        except:

                                               pass

                for filename in filenames:

                    localpath = os.path.join(local, dirpath, filename)

                    remotepath = os.path.join(parent, dirpath, filename)

                    self.sftp.get(remotepath, localpath)

        self.scp.close()

if __name__=='__main__':

    print "-"*50

    getthread=get_thread()

    room_pathd = '/opt/src/logs/'

    room_paths = '/home/python/'

    getthread.getall(room_paths,room_pathd)

因为想作为类，然后读取文件，批量传输改进版

#!/usr/bin/python   

import pexpect

import paramiko  

import os

import sys

import time

import multiprocessing

import datetime

import crash_on_ipy

from stat import S_ISDIR

ip_list = []
#room_id = sys.argv[1]

cur_time = datetime.datetime.now()

class get_thread():

    def __init__(self,hostname,password,username,port=None):

   #def __init__(self,hostname,username='root',key_file=None,password=None): 

   #def __init__(self,hostname=None,password=None,username=None,port=None,local_dir=None,remote_dir=None):

        self.hostname = hostname

        self.username = username

        self.password = password

        self.scp = paramiko.SSHClient()

        self.scp.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        self.scp = paramiko.Transport((hostname, 22))

        self.scp.connect(username=username, password=password)

        self.sftp = paramiko.SFTPClient.from_transport(self.scp)

    def _walk_remote(self, dirpath):

        dirnames = []

        filenames = []
 
        for fd in self.sftp.listdir_attr(dirpath):

            if S_ISDIR(fd.st_mode):

                dirnames.append(fd.filename)

            else:

                filenames.append(fd.filename)

        yield dirpath, dirnames, filenames
 
        for dirname in dirnames:

            new_dirpath = os.path.join(dirpath, dirname)

            # yield from self._walk_remote(new_dirpath)

            for walk in self._walk_remote(new_dirpath):

                yield walk
 
    def getall(self,local,remote):

        st_mode = self.sftp.stat(remote).st_mode

        if not S_ISDIR(st_mode):

            filename = os.path.basename(remote)

            self.sftp.get(remote, os.path.join(local, filename))

        else:

            parent, child = os.path.split(remote)
 
            for dirpath, dirnames, filenames in self._walk_remote(remote):

                dirpath = dirpath.replace(parent, '.')

                for dirname in dirnames:

                    try:

                        os.makedirs(os.path.join(local, dirpath, dirname))

                    except:

                        pass
 
                for filename in filenames:

                    localpath = os.path.join(local, dirpath, filename)

                    remotepath = os.path.join(parent, dirpath, filename)

                    self.sftp.get(remotepath, localpath)

        self.scp.close()

if __name__=='__main__':

    print "-"*50

    #getthread=get_thread()

    #room_pathd = '/opt/src/logs/'

    #room_paths = '/home/python/'

    f = file('/home/filelist','r')

    c = f.readlines()

    for x in c:

        hostname = x.split('::')[0]

        password = x.split('::')[1]

        username = x.split('::')[2]

        local= x.split('::')[3].strip('\n')

        remote = x.split('::')[4].strip('\n')

        print local,remote        

        getthread=get_thread(hostname,password,username)

        getthread.getall(local,remote)

    f.close()

   #getthread.getall(room_paths,room_pathd)

最后，又想实现多进程，但是碰到困难了，没有实现，报错

UnpickleableError: Cannot pickle <type 'thread.lock'> objects

报错版不可用，待改进

#!/usr/bin/python   

import pexpect

import paramiko  

import os

import sys

import time

import multiprocessing

import datetime

import crash_on_ipy

from stat import S_ISDIR

import copy_reg

import types

ip_list = []
#room_id = sys.argv[1]

cur_time = datetime.datetime.now()

class get_thread():

    def __init__(self,hostname='ip'):

        self.hostname='ip'

        self.scp = paramiko.SSHClient()

        self.scp.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        self.scp = paramiko.Transport((hostname, 22))

        self.scp.connect(username='tomcat', password='xx')

        self.sftp = paramiko.SFTPClient.from_transport(self.scp)

    def walk_remote(self, dirpath):

        dirnames = []

        filenames = []
 
        for fd in self.sftp.listdir_attr(dirpath):

            if S_ISDIR(fd.st_mode):

                dirnames.append(fd.filename)

            else:

                filenames.append(fd.filename)

        yield dirpath, dirnames, filenames
 
        for dirname in dirnames:

            new_dirpath = os.path.join(dirpath, dirname)

            # yield from self.walk_remote(new_dirpath)

            for walk in self.walk_remote(new_dirpath):

                yield walk
 
    def __call__(self,local,remote):

        return self.getall(local,remote)
 
    def getall(self,local,remote):

        st_mode = self.sftp.stat(remote).st_mode

        if not S_ISDIR(st_mode):

            filename = os.path.basename(remote)

            self.sftp.get(remote, os.path.join(local, filename))

        else:

            parent, child = os.path.split(remote)
 
            for dirpath, dirnames, filenames in self.walk_remote(remote):

                dirpath = dirpath.replace(parent, '.')

                for dirname in dirnames:

                    try:

                        os.makedirs(os.path.join(local, dirpath, dirname))

                    except:

                        pass
 
                for filename in filenames:

                    localpath = os.path.join(local, dirpath, filename)

                    remotepath = os.path.join(parent, dirpath, filename)

                    self.sftp.get(remotepath, localpath)

        self.scp.close()
 
    def run(self):

        room_pathd = '/opt/src/logs/'

        room_paths = '/home/python/'

        pool = multiprocessing.Pool(processes=1)

        pool.apply_async(self, args=(room_paths,room_pathd))

        #pool.apply_async(self.getall,args=(room_paths,room_pathd,))

        #pool.apply_async(proxy, args=(self, num,))

        #pool.apply_async(self.func, args=(num,))

        pool.close()

        pool.join()
 
if __name__=='__main__':

    print "-"*50

    getthread=get_thread()

    getthread.run()

本文转自 liqius 51CTO博客，原文链接：http://blog.51cto.com/szgb17/1860408，如需转载请自行联系原作者

python sftp.get 同步远程文件夹 实践二

python sftp.get 同步远程文件夹实践二