一直以来,想写一个多线程下载的程序。苦于没得时间,前短时间连续阴雨天,打开电脑百无聊奈。
就把这个程序写了出来。现在发出来,供大家参考。
首先,做分段下载,那么我们得分清楚每一块的数据格式。下图是多线程分段下载的示例图。
1:分段文件块的类
-
package com.eibit.javalearning;
-
-
import java.io.Serializable;
-
-
/**
-
* Created by Administrator on 2016/4/15.
-
*/
-
public class BlockBean implements Serializable{
-
private static final long serialVersionUID = 1L;
-
//最大的下载尝试次数
-
public static final int MAX_TRY_COUNT = 10;
-
/**
-
* 每次尝试下载的次数
-
*/
-
public int tryCount = 1;
-
/**
-
* 偏移量
-
*/
-
public long offset = 0;
-
/**
-
* 分配给本线程的下载字节数
-
*/
-
public long length = 0;
-
/**
-
* 已下载的字节数
-
*/
-
public long downloadedLength = 0;
-
/**
-
* block的编号
-
*/
-
public int i;
-
-
public BlockBean(int i, long offset, long length) {
-
this.i = i;
-
this.offset = offset;
-
this.length = length;
-
this.downloadedLength = 0;
-
}
-
}
-
复制代码
2:下载进度监听接口
-
package com.eibit.javalearning;
-
-
public interface DownListener {
-
-
/**
-
* 开始下载
-
*/
-
void onStartDownLoad();
-
-
/**
-
* 下载进度改变
-
* @param completeRate 下载进度
-
* @param speed 下载速度 KB/S
-
*/
-
void onCompleteRateChanged(int completeRate);
-
-
/**
-
* 下载完成
-
* @param fileName 存储的文件名
-
*/
-
void onDownloadCompleted(String fileName);
-
-
/**
-
* 开启多线程下载
-
* @param numberThread 线程数量
-
*/
-
void onStartMultiDownLoad(int numberThread);
-
-
/**
-
* 当某块下载完成的时候
-
* @param blockNum 块号
-
* @param readBytes 下载完成的字节
-
*/
-
void onBlockCompleted(int blockNum, long readBytes);
-
-
/**
-
* 下载速度
-
* @param speed 下载速度
-
*/
-
void onDownLoadSpeed(long speed);
-
}
-
复制代码
3: 多线程下载控制类
-
package com.eibit.javalearning;
-
-
import java.io.BufferedInputStream;
-
import java.io.File;
-
import java.io.FileOutputStream;
-
import java.io.IOException;
-
import java.io.InputStream;
-
import java.io.RandomAccessFile;
-
import java.net.HttpURLConnection;
-
import java.net.MalformedURLException;
-
import java.net.URL;
-
import java.util.LinkedList;
-
import java.util.concurrent.ExecutorService;
-
import java.util.concurrent.Executors;
-
-
import javax.net.ssl.HttpsURLConnection;
-
-
/**
-
* 文件下载管理类
-
*/
-
public class ThreadPoolMultiDownLoadManager {
-
private String path; //资源的原始路径比如说是http://dlsw.baidu.com/sw-search-sp/soft/ea/12585/mysql-5.6.24-win32.1432006610.zip
-
private String targetFile; //下载的文件存储的目标地址
-
private DownListener downListener; //下载的监听器
-
-
private static long blockSize = 1 * 1024 * 1024; //每块的大小,默认是1MB
-
-
//最大线程数量
-
private int MAX_THREAD_NUM = 5; //线程池的线程的数量
-
-
private long fileSize; //文件的总大小
-
//private long mTotalDownloadedBytes = 0; //总的已下载的字节数
-
private long mLastDownloadedTime = 0; //上次的已下载的字节数的时间,为毫秒,主要用于下载速度的
-
private int lastDownloadPercent = 0; //上次下载的百分比,主要是用于计算下载的百分比的
-
//固定容量大小的线程池
-
private ExecutorService executorService;
-
-
private final Object mLock = new Object(); //用于块队列的同步的锁,不能让一块文件被两个线程下载
-
//private final Object lock = new Object(); //用于计算下载百分比同步锁
-
private LinkedList<BlockBean> mQueue = new LinkedList<BlockBean>(); //块队列
-
private int mCompletedBlock = 0; //已完成的块数
-
private int blockCount = 0; //总的块数
-
private int mCompletedThreadCount = 0; //已完成的线程的数量,用于统计是否下载结束
-
private long mDeltaDownLoadedBytes = 0; //单位时间内下载的字节数
-
-
/**
-
* 构造器
-
* @param path 资源路径
-
* @param targetFile 目标文件的存放地址
-
* @param downListener 下载监听器
-
*/
-
public ThreadPoolMultiDownLoadManager(String path, String targetFile, DownListener downListener){
-
this.path = path;
-
this.targetFile = targetFile;
-
this.downListener = downListener;
-
}
-
-
/**
-
* 开启多线程下载
-
*/
-
public void multiDownload(){
-
//初始化,进行清零操作
-
lastDownloadPercent = 0;
-
//mTotalDownloadedBytes = 0;
-
mCompletedBlock = 0;
-
mCompletedThreadCount = 0;
-
//初始化线程池
-
executorService = Executors.newFixedThreadPool(MAX_THREAD_NUM);
-
//执行一个任务
-
//executorService.execute(new GetContentThread());
-
new Thread(new GetContentThread()).start(); //不把这个任务放到线程池里面去
-
}
-
-
/**
-
* 获取文件的大小和属性的一个总控线程
-
*/
-
private class GetContentThread implements Runnable {
-
@Override
-
public void run() {
-
//开始下载
-
if(downListener != null) {
-
downListener.onStartDownLoad();
-
}
-
doDownload(path, targetFile);
-
}
-
}
-
-
/**
-
* 计算文件大小, 并开启线程下载
-
* @param path 资源的url地址
-
* @param targetFile 资源本地存放路径
-
*/
-
private void doDownload(String path, String targetFile){
-
try {
-
HttpURLConnection httpConnection = (HttpURLConnection) new URL(path).openConnection();
-
//只获取文件的响应报头,而不获取文件的内容,这个响应报头里面就会包含文件大小等信息
-
httpConnection.setRequestMethod("HEAD");
-
httpConnection.connect();
-
-
int responseCode = httpConnection.getResponseCode();
-
System.out.println("---------------------------------------responseCode=" + responseCode);
-
if(responseCode >= 400){
-
System.out.println("Web服务器响应错误!");
-
return;
-
}
-
httpConnection.disconnect();
-
if(responseCode == HttpsURLConnection.HTTP_OK) { //服务器响应成功
-
//获取文件的大小
-
fileSize = httpConnection.getContentLength();
-
if(fileSize <0) { //使用HEAD获取不到文件大小
-
UnKnownSizeSingleDowloadThread thread = new UnKnownSizeSingleDowloadThread(path, targetFile);
-
executorService.execute(thread);
-
executorService.shutdown();
-
} else { //可以获取到文件的大小
-
//生成文件
-
File newFile = new File(targetFile);
-
//生成一个可以随机存取的文件RandomAccessFile,以读写的方式打开
-
RandomAccessFile raf = new RandomAccessFile(newFile, "rw");
-
raf.setLength(fileSize); //设置这个文件大小
-
raf.close(); //关闭这个文件
-
-
blockCount = 0; //块的数量
-
//计算有多少个块
-
if(fileSize % blockSize == 0) {
-
blockCount = (int) (fileSize / blockSize);
-
} else {
-
blockCount = (int)(fileSize /blockSize) + 1;
-
}
-
System.out.println("Block Count = "+ blockCount);
-
-
mLastDownloadedTime = System.currentTimeMillis(); //初始化第一次下载时间为毫秒级
-
//再小实际上都可以用块进行下载,可以将其判断去掉
-
if(blockCount <=5 ){ //如果块的数量小于5块,没必要使用多线程进行下载
-
SingleDowloadThread singleDowloadThread = new SingleDowloadThread(path, targetFile, fileSize);
-
executorService.execute(singleDowloadThread);
-
executorService.shutdown(); //关闭线程池
-
} else { //如果块的数量大于5块,则开启多线程下载
-
//初始化块的队列
-
long offset = 0;
-
for(int i=0; i< blockCount; i++) {
-
//最后的一块需要特殊对待,因为最后一块的大小可能没有blockSize这么大
-
if(i == blockCount -1) {
-
BlockBean bean = new BlockBean(i, offset, fileSize-offset);
-
offset += blockSize;
-
mQueue.add(bean);
-
} else {
-
BlockBean bean = new BlockBean(i, offset, blockSize);
-
offset += blockSize;
-
mQueue.add(bean);
-
}
-
}
-
//使用线程池进行操作
-
for(int i = 0; i< MAX_THREAD_NUM; i++) {
-
DownloadTask task = new DownloadTask(path, targetFile);
-
executorService.execute(task);
-
}
-
executorService.shutdown();
-
}
-
}
-
}
-
} catch (MalformedURLException e) {
-
e.printStackTrace();
-
} catch (IOException e) {
-
e.printStackTrace();
-
}
-
}
-
-
/**
-
* 负责文件下载的类
-
*/
-
private class DownloadTask implements Runnable {
-
private String url = null;
-
private String fileName = null;
-
-
protected DownloadTask(String url, String fileName) {
-
this.url = url;
-
this.fileName = fileName;
-
}
-
-
public void run() {
-
boolean flag = false; //是否还有块没有下载完成
-
while(!flag) {
-
BlockBean block = null;
-
//首先去查找还有没有块已完成
-
synchronized (mLock) {
-
if(mQueue.size() > 0) {
-
//取一块出来
-
block = mQueue.remove();
-
}
-
}
-
-
//测试代码
-
/*if(block != null) {
-
System.out.println("block=" + block.i + ", \tThread Number=" + Thread.currentThread().getName());
-
try {
-
Thread.sleep(200);
-
} catch (InterruptedException e) {
-
e.printStackTrace();
-
}
-
-
if(block.i % 20 == 0) {
-
System.out.println("IOException block =" + block.i);
-
synchronized (lock) {
-
if(block.tryCount++ < BlockBean.MAX_TRY_COUNT) { //如果还没有达到最大的尝试次数,那么就接着继续下载
-
mQueue.add(block); //把这一块重新加到队列里面去
-
}
-
}
-
}
-
} else {
-
flag = true;
-
}*/
-
-
if(block != null) { //还有块没有下载完成
-
long offset = block.offset;
-
long length = block.length;
-
int i = block.i;
-
//long mLastDownloadedTime = System.currentTimeMillis();
-
//System.out.println("downLoad block i=" + i + " finished");
-
try {
-
HttpURLConnection httpConnection = (HttpURLConnection) new URL(url).openConnection();
-
httpConnection.setRequestMethod("GET");
-
httpConnection.setRequestProperty("Accept", "image/gif, image/jpeg, image/pjpeg,*/*");
-
httpConnection.setRequestProperty("Accept-Language", "zh_CN");
-
httpConnection.setRequestProperty("Charset", "UTF-8");
-
httpConnection.setRequestProperty("User-Agent", "Mozilla/4.0(compatible; MSIE 7.0; Windows NT 5.2;)");
-
httpConnection.setRequestProperty("Connection", "Keep-Alive");
-
//设置RANGE属性,向服务器请求,其中的某一段数据
-
httpConnection.setRequestProperty("RANGE", "bytes=" + offset + "-" + (offset + length - 1));
-
httpConnection.setConnectTimeout(15000); //15秒的连接超时
-
httpConnection.setReadTimeout(30000);//30秒的读写超时,连接和读写超时跟网络状态有关
-
int responseCode = httpConnection.getResponseCode();
-
if(responseCode == HttpURLConnection.HTTP_PARTIAL) { //部分内容
-
BufferedInputStream bis = new BufferedInputStream(httpConnection.getInputStream());
-
byte[] buffer = new byte[1024];
-
int readedBytes = 0;
-
File newFile = new File(fileName);
-
RandomAccessFile raf = new RandomAccessFile(newFile, "rw");
-
raf.seek(offset); //跳转到文件的指定位置
-
while((readedBytes = bis.read(buffer,0, buffer.length)) != -1) {
-
raf.write(buffer, 0, readedBytes);
-
//currentDownloadBytes += readedBytes;
-
}
-
raf.close();
-
bis.close();
-
httpConnection.disconnect();
-
mDeltaDownLoadedBytes+= length; //单位时间内下载的字节数增加
-
-
//以下代码已经废弃
-
/*synchronized (lock) {
-
mTotalDownloadedBytes += length;
-
//计算现在速度
-
long currentTime = System.currentTimeMillis();
-
//速度是多少KB/s
-
long speed = (long) (length / ((currentTime - mLastDownloadedTime) /1000.0) /1024);
-
mLastDownloadedTime = currentTime; //重新初始化时间
-
-
//计算下载的百分比
-
int downloadPercent = (int) (mTotalDownloadedBytes * 100 / fileSize);
-
//System.out.println("mTotalDownloadedBytes=" + mTotalDownloadedBytes + ",fileSize=" + fileSize + ",downloadPercent=" + downloadPercent);
-
if(downloadPercent -lastDownloadPercent >= 1) {//百分之百可能不通知
-
//保证每次更新进展值都大于1,避免更新太过于频繁
-
lastDownloadPercent = downloadPercent;
-
if(downListener != null) {
-
downListener.onCompleteRateChanged(downloadPercent,speed);
-
}
-
}
-
if(downloadPercent == 100) {
-
if(downListener != null) {
-
downListener.onCompleteRateChanged(downloadPercent, speed);
-
}
-
}
-
}*/
-
if(downListener != null) {
-
downListener.onBlockCompleted(i, length);
-
synchronized (mLock) {
-
mCompletedBlock++; //已完成的块加1
-
int percent = mCompletedBlock * 100 / blockCount;
-
downListener.onCompleteRateChanged(percent);
-
long currentTime = System.currentTimeMillis();
-
-
if((currentTime - mLastDownloadedTime ) /1000 > 1) {//时间差不到1秒不要去计算速度
-
//速度是多少KB/s
-
long speed = (long) (mDeltaDownLoadedBytes / ((currentTime - mLastDownloadedTime) /1000.0) /1024);
-
mDeltaDownLoadedBytes = 0; //清空单位时间内下载的字节数
-
mLastDownloadedTime = currentTime; //重新初始化时间
-
downListener.onDownLoadSpeed(speed);
-
}
-
}
-
}
-
}
-
} catch (IOException e) {
-
e.printStackTrace();
-
//如果出现IO异常,则表示这一块的数据没有完整的下载下来
-
//那么将进行一次重新下载
-
synchronized (mLock) {
-
System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<< IOException blockNumber:" + i + "<<<<<<<<<<<<<<<<<<<<<<<<");
-
blockCount++; //出现异常一次,需要下载的块的数量加1
-
if(block.tryCount++ < BlockBean.MAX_TRY_COUNT) { //如果还没有达到最大的尝试次数,那么就接着继续下载
-
mQueue.add(block); //把这一块重新加到队列里面去
-
}
-
}
-
}
-
} else {//没有块可供下载,则结束掉当前线程
-
flag = true;
-
break;
-
}
-
}
-
/*if(mTotalDownloadedBytes >= totalLength && downListener != null) { //下载结束
-
downListener.onDownloadCompleted(targetFile);
-
}*/
-
System.out.println("------------------------------"+ Thread.currentThread().getName() + "----------------------" );
-
synchronized (mLock) {
-
mCompletedThreadCount++; //已完成的线程数量加1
-
if(downListener != null){
-
if(mCompletedThreadCount == MAX_THREAD_NUM){ //已完成的线程数如果大于等于线程池的大小,则下载完成
-
downListener.onDownloadCompleted(fileName);
-
}
-
}
-
}
-
}
-
}
-
-
private class SingleDowloadThread implements Runnable{
-
private String url = null;
-
private String fileName = null;
-
private long fileSize = 0; //下载的总长度
-
-
public SingleDowloadThread(String url, String fileName, long fileSize) {
-
this.url = url;
-
this.fileName = fileName;
-
this.fileSize = fileSize;
-
}
-
-
@Override
-
public void run() {
-
HttpURLConnection httpConnection;
-
try {
-
httpConnection = (HttpURLConnection) new URL(url).openConnection();
-
httpConnection.setRequestMethod("GET");
-
httpConnection.setRequestProperty("Accept", "image/gif, image/jpeg, image/pjpeg,*/*");
-
httpConnection.setRequestProperty("Accept-Language", "zh_CN");
-
httpConnection.setRequestProperty("Charset", "UTF-8");
-
httpConnection.setRequestProperty("User-Agent", "Mozilla/4.0(compatible; MSIE 7.0; Windows NT 5.2;)");
-
httpConnection.setRequestProperty("Connection", "Keep-Alive");
-
httpConnection.setConnectTimeout(15000); //15秒的连接超时
-
httpConnection.setReadTimeout(30000);//30秒的读写超时
-
int responseCode = httpConnection.getResponseCode();
-
if(responseCode == HttpURLConnection.HTTP_OK) {
-
InputStream inputStream = httpConnection.getInputStream();
-
FileOutputStream outputStream = new FileOutputStream(fileName);
-
byte[] buffer = new byte[1024];
-
int len = -1;
-
long readedBytes = 0;
-
long deltaReadBytes = 0; //在1秒的时间间隔之内,下载的字节数
-
//long mLastDownloadedTime = System.currentTimeMillis();
-
while( (len = inputStream.read(buffer, 0, buffer.length)) != -1) {
-
outputStream.write(buffer, 0, len);
-
readedBytes += len;
-
deltaReadBytes += len;
-
-
long currentTime = System.currentTimeMillis();
-
//System.out.println("currentTime - mLastDownloadedTime=" + (currentTime - mLastDownloadedTime) + "len=" + len);
-
if((currentTime - mLastDownloadedTime ) /1000 > 1) {//时间差不到1秒不要去计算速度
-
long speed = (long) (deltaReadBytes / ((currentTime - mLastDownloadedTime) / 1000.0) /1024);
-
deltaReadBytes = 0; //计算了速度之后,需要对deltaReadBytes清零
-
mLastDownloadedTime = currentTime;
-
if(downListener != null) {
-
downListener.onDownLoadSpeed(speed);
-
}
-
}
-
//计算百分比
-
int percent = (int) (readedBytes * 100 / fileSize);
-
if(downListener != null) {
-
if(percent - lastDownloadPercent >= 1) {
-
downListener.onCompleteRateChanged(percent);
-
}
-
}
-
}
-
if(downListener != null) {
-
downListener.onDownloadCompleted(fileName);
-
}
-
inputStream.close();
-
outputStream.close();
-
}
-
} catch (MalformedURLException e) {
-
e.printStackTrace();
-
} catch (IOException e) {
-
e.printStackTrace();
-
}
-
}
-
-
}
-
-
private class UnKnownSizeSingleDowloadThread implements Runnable{
-
private String url = null;
-
private String fileName = null;
-
-
public UnKnownSizeSingleDowloadThread(String url, String fileName) {
-
this.url = url;
-
this.fileName = fileName;
-
}
-
-
@Override
-
public void run() {
-
HttpURLConnection httpConnection;
-
try {
-
httpConnection = (HttpURLConnection) new URL(url).openConnection();
-
httpConnection.setRequestMethod("GET");
-
httpConnection.setRequestProperty("Accept", "image/gif, image/jpeg, image/pjpeg,*/*");
-
httpConnection.setRequestProperty("Accept-Language", "zh_CN");
-
httpConnection.setRequestProperty("Charset", "UTF-8");
-
httpConnection.setRequestProperty("User-Agent", "Mozilla/4.0(compatible; MSIE 7.0; Windows NT 5.2;)");
-
httpConnection.setRequestProperty("Connection", "Keep-Alive");
-
httpConnection.setConnectTimeout(15000); //15秒的连接超时
-
httpConnection.setReadTimeout(30000);//30秒的读写超时
-
int responseCode = httpConnection.getResponseCode();
-
if(responseCode == HttpURLConnection.HTTP_OK) {
-
InputStream inputStream = httpConnection.getInputStream();
-
FileOutputStream outputStream = new FileOutputStream(fileName);
-
byte[] buffer = new byte[1024];
-
int len = -1;
-
long deltaReadBytes = 0; //在1秒的时间间隔之内,下载的字节数
-
//long mLastDownloadedTime = System.currentTimeMillis();
-
while( (len = inputStream.read(buffer, 0, buffer.length)) != -1) {
-
outputStream.write(buffer, 0, len);
-
deltaReadBytes += len;
-
-
long currentTime = System.currentTimeMillis();
-
//System.out.println("currentTime - mLastDownloadedTime=" + (currentTime - mLastDownloadedTime) + "len=" + len);
-
if((currentTime - mLastDownloadedTime ) /1000 > 1) {//时间差不到1秒不要去计算速度
-
long speed = (long) (deltaReadBytes / ((currentTime - mLastDownloadedTime) / 1000.0) /1024);
-
deltaReadBytes = 0; //计算了速度之后,需要对deltaReadBytes清零
-
mLastDownloadedTime = currentTime;
-
if(downListener != null) {
-
downListener.onDownLoadSpeed(speed);
-
}
-
}
-
//大小未知,无法计算百分比
-
}
-
if(downListener != null) {
-
downListener.onDownloadCompleted(fileName);
-
}
-
inputStream.close();
-
outputStream.close();
-
}
-
} catch (MalformedURLException e) {
-
e.printStackTrace();
-
} catch (IOException e) {
-
e.printStackTrace();
-
}
-
}
-
-
}
-
}
复制代码
4: 下载程序测试主类。
-
package com.eibit.javalearning;
-
-
/**
-
* Created by Administrator on 2016/4/16.
-
*/
-
public class TestMultiDownload implements DownListener{
-
-
public static void main(String[] args) {
-
String URL = "http://123.147.165.57:9999/dlsw.baidu.com/sw-search-sp/soft/ea/12585/mysql-5.6.24-win32.1432006610.zip";
-
String TARGET_FILE ="E:\\mysql-5.6.24-win32.1432006610.zip";
-
TestMultiDownload download = new TestMultiDownload();
-
ThreadPoolMultiDownLoadManager downLoadManager = new ThreadPoolMultiDownLoadManager(URL, TARGET_FILE, download);
-
downLoadManager.multiDownload();
-
}
-
-
@Override
-
public void onStartDownLoad() {
-
System.out.println("************ onStartDownLoad **************");
-
}
-
-
@Override
-
public void onCompleteRateChanged(int completeRate) {
-
System.out.println("************ onCompleteRateChanged completeRate= " + completeRate);
-
}
-
-
@Override
-
public void onDownloadCompleted(String fileName) {
-
System.out.println("----------------------- onDownloadCompleted fileName = " + fileName + "-------------------");
-
}
-
-
@Override
-
public void onStartMultiDownLoad(int numberThread) {
-
System.out.println("----------------------- onStartMultiDownLoad numberThread = " + numberThread + "-------------------");
-
}
-
-
@Override
-
public void onBlockCompleted(int blockNum, long readBytes) {
-
System.out.println("----------------------- onBlockCompleted BlockNum = " + blockNum + ",readBytes=" + readBytes + "-------------------");
-
}
-
-
@Override
-
public void onDownLoadSpeed(long speed) {
-
System.out.println("----------------------- onDownLoadSpeed speed=" + speed + " KB/s");
-
}
-
}
-
复制代码
匆忙之间写下的程序,其中肯定有不足之处。欢迎大家不吝赐教! |