数据库查出数据集合直接生成txt,excel文件上送到Sftp服务器
1、sftp上送需要的依赖
<!-- sft文件操作 -->
<dependency>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
<version>0.1.54</version>
</dependency>
2、sftp上送文件类
import com.allinpay.datamanager.entity.SftpConfig;
import com.jcraft.jsch.*;
import com.jcraft.jsch.ChannelSftp.LsEntry;
import lombok.extern.slf4j.Slf4j;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.Vector;
@Slf4j
public class SftpUtils {
private long count;
/**
* 已经连接次数
*/
private long count1 = 0;
/**
*重试间隔时间
*/
private long sleepTime;
public SftpUtils(long count, long sleepTime) {
this.count = count;
this.sleepTime = sleepTime;
}
public SftpUtils() {
}
/**
* 连接sftp服务器
*
* @return
*/
public ChannelSftp connect(SftpConfig sftpConfig) {
ChannelSftp sftp = null;
try {
JSch jsch = new JSch();
jsch.getSession(sftpConfig.getUsername(), sftpConfig.getHostname(), Integer.valueOf(sftpConfig.getPort()));
Session sshSession = jsch.getSession(sftpConfig.getUsername(), sftpConfig.getHostname(),
Integer.valueOf(sftpConfig.getPort()));
log.info("Session created ... UserName=" + sftpConfig.getUsername() + ";host=" + sftpConfig.getHostname()
+ ";port=" + sftpConfig.getPort());
sshSession.setPassword(sftpConfig.getPassword());
Properties sshConfig = new Properties();
sshConfig.put("StrictHostKeyChecking", "no");
sshSession.setConfig(sshConfig);
sshSession.connect();
log.info("Session connected ...");
log.info("Opening Channel ...");
Channel channel = sshSession.openChannel("sftp");
channel.connect(60000);
sftp = (ChannelSftp) channel;
log.info("登录成功");
} catch (Exception e) {
try {
count1 += 1;
if (count == count1) {
throw new RuntimeException(e);
}
Thread.sleep(sleepTime);
log.info("重新连接....");
connect(sftpConfig);
} catch (InterruptedException e1) {
throw new RuntimeException(e1);
}
}
return sftp;
}
/**
* 上传文件
*
* @param directory 上传的目录
* @param uploadFile 要上传的文件
* @param sftpConfig
*/
public void upload(String directory, String uploadFile, SftpConfig sftpConfig) {
ChannelSftp sftp = connect(sftpConfig);
try {
sftp.cd(directory);
} catch (SftpException e) {
try {
// 目录不存在,则创建文件夹
String[] dirs = directory.split("/");
String tempPath = "";
for (String dir : dirs) {
if (null == dir || "".equals(dir))
continue;
tempPath += "/" + dir;
try {
sftp.cd(tempPath);
} catch (SftpException ex) {
sftp.mkdir(tempPath);
sftp.cd(tempPath);
} catch (Exception e1) {
log.error("sftp上传目录创建失败", e1);
}
}
} catch (SftpException e1) {
throw new RuntimeException("ftp创建文件路径失败" + directory);
}
}
File file = new File(uploadFile);
InputStream inputStream = null;
try {
inputStream = new FileInputStream(file);
sftp.put(inputStream, file.getName());
} catch (Exception e) {
throw new RuntimeException("sftp异常" + e);
} finally {
disConnect(sftp);
closeStream(inputStream, null);
}
}
/**
* 上传文件
*
* @param directory 上传的目录
* @param fileName 要上传的文件名
* @param inputStream 上传的文件流
* @param sftpConfig
*/
public void upload(String directory, String fileName, InputStream inputStream, SftpConfig sftpConfig) {
ChannelSftp sftp = connect(sftpConfig);
try {
sftp.cd(directory);
} catch (SftpException e) {
try {
// 目录不存在,则创建文件夹
String[] dirs = directory.split("/");
String tempPath = "";
for (String dir : dirs) {
if (null == dir || "".equals(dir))
continue;
tempPath += "/" + dir;
try {
sftp.cd(tempPath);
} catch (SftpException ex) {
sftp.mkdir(tempPath);
sftp.cd(tempPath);
} catch (Exception e1) {
log.error("sftp上传目录创建失败", e1);
}
}
} catch (SftpException e1) {
throw new RuntimeException("ftp创建文件路径失败" + directory);
}
}
try {
sftp.put(inputStream, fileName);
} catch (Exception e) {
throw new RuntimeException("sftp异常" + e);
} finally {
disConnect(sftp);
closeStream(inputStream, null);
}
}
/**
* 下载文件
*
* @param directory 下载目录
* @param downloadFile 下载的文件
* @param saveFile 存在本地的路径
* @param sftpConfig
*/
public void download(String directory, String downloadFile, String saveFile, SftpConfig sftpConfig) {
OutputStream output = null;
try {
File localDirFile = new File(saveFile);
// 判断本地目录是否存在,不存在需要新建各级目录
if (!localDirFile.exists()) {
localDirFile.mkdirs();
}
if (log.isInfoEnabled()) {
log.info("开始获取远程文件:[{}]---->[{}]", new Object[]{directory, saveFile});
}
ChannelSftp sftp = connect(sftpConfig);
sftp.cd(directory);
if (log.isInfoEnabled()) {
log.info("打开远程文件:[{}]", new Object[]{directory});
}
output = new FileOutputStream(new File(saveFile.concat(File.separator).concat(downloadFile)));
sftp.get(downloadFile, output);
if (log.isInfoEnabled()) {
log.info("文件下载成功");
}
disConnect(sftp);
} catch (Exception e) {
if (log.isInfoEnabled()) {
log.info("文件下载出现异常,[{}]", e);
}
throw new RuntimeException("文件下载出现异常,[{}]", e);
} finally {
closeStream(null, output);
}
}
/**
* 下载远程文件夹下的所有文件
*
* @param remoteFilePath
* @param localDirPath
* @throws Exception
*/
public void getFileDir(String remoteFilePath, String localDirPath, SftpConfig sftpConfig) throws Exception {
File localDirFile = new File(localDirPath);
// 判断本地目录是否存在,不存在需要新建各级目录
if (!localDirFile.exists()) {
localDirFile.mkdirs();
}
if (log.isInfoEnabled()) {
log.info("sftp文件服务器文件夹[{}],下载到本地目录[{}]", new Object[]{remoteFilePath, localDirFile});
}
ChannelSftp channelSftp = connect(sftpConfig);
Vector<LsEntry> lsEntries = channelSftp.ls(remoteFilePath);
if (log.isInfoEnabled()) {
log.info("远程目录下的文件为[{}]", lsEntries);
}
for (LsEntry entry : lsEntries) {
String fileName = entry.getFilename();
if (checkFileName(fileName)) {
continue;
}
String remoteFileName = getRemoteFilePath(remoteFilePath, fileName);
channelSftp.get(remoteFileName, localDirPath);
}
disConnect(channelSftp);
}
/**
* 关闭流
*
* @param outputStream
*/
private void closeStream(InputStream inputStream, OutputStream outputStream) {
if (outputStream != null) {
try {
outputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
private boolean checkFileName(String fileName) {
if (".".equals(fileName) || "..".equals(fileName)) {
return true;
}
return false;
}
private String getRemoteFilePath(String remoteFilePath, String fileName) {
if (remoteFilePath.endsWith("/")) {
return remoteFilePath.concat(fileName);
} else {
return remoteFilePath.concat("/").concat(fileName);
}
}
/**
* 删除文件
*
* @param directory 要删除文件所在目录
* @param deleteFile 要删除的文件
* @param sftp
*/
public void delete(String directory, String deleteFile, ChannelSftp sftp) {
try {
sftp.cd(directory);
sftp.rm(deleteFile);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* 列出目录下的文件
*
* @param directory 要列出的目录
* @param sftpConfig
* @return
* @throws SftpException
*/
public List<String> listFiles(String directory, SftpConfig sftpConfig) throws SftpException {
ChannelSftp sftp = connect(sftpConfig);
List fileNameList = new ArrayList();
try {
sftp.cd(directory);
} catch (SftpException e) {
return fileNameList;
}
Vector vector = sftp.ls(directory);
for (int i = 0; i < vector.size(); i++) {
if (vector.get(i) instanceof LsEntry) {
LsEntry lsEntry = (LsEntry) vector.get(i);
String fileName = lsEntry.getFilename();
if (".".equals(fileName) || "..".equals(fileName)) {
continue;
}
fileNameList.add(fileName);
}
}
disConnect(sftp);
return fileNameList;
}
/**
* 断掉连接
*/
public void disConnect(ChannelSftp sftp) {
try {
sftp.disconnect();
sftp.getSession().disconnect();
} catch (Exception e) {
e.printStackTrace();
}
}
}
3、数据库查询数据生成Excel到Sftp服务器
实体类
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.excel.metadata.BaseRowModel;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
public class PeopleExcelModal {
/**
* 姓名
*/
@ExcelProperty(value="姓名",index=1)
private String name;
/**
* 年龄
*/
@ExcelProperty(value="年龄",index=2)
private String age;
/**
* 联系电话
*/
@ExcelProperty(value="联系电话",index=3)
private String Phone;
/**
* 地址
*/
@ExcelProperty(value="地址",index=4)
private String address;
}
查询上送
private void uploadToSftp() {
String fileName = "people";
//sftp服务器的ip地址
String sftpIp = "127.0.0.1";
//sftp服务器的端口,一般默认22
String sftpPort = "22";
//sftp服务器的账号
String sftpAccount = "sftpAcc";
//sftp服务器的密码
String sftpPassWord = "sftpPw";
SftpConfig sftpConfig = new SftpConfig(sftpIp, sftpPort,
sftpAccount, sftpPassWord, "60000");
//查询所有people数据
List<PeopleExcelModal> peopleList = peopleMapper.queryAll();
//创建一个流,等待写入excel文件内容
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
//将excel文件写入byteArrayOutputStream中
EasyExcel.write(byteArrayOutputStream, PeopleExcelModal.class)
.registerWriteHandler(new CustomRowWriteHandler())
.registerWriteHandler(new CustomCellWeightWeightConfig())
.sheet(fileName).doWrite(peopleList);
SftpUtils sftpUtils = new SftpUtils(3, 6000);
//创建inputStream流
InputStream inputStream = new ByteArrayInputStream(byteArrayOutputStream.toByteArray());
sftpUtils.upload(sftpFileUrl, fileName + ".xlsx", inputStream, sftpConfig);
}
CustomRowWriteHandler 类在excel的每行数据前生成序号
import com.alibaba.excel.write.handler.RowWriteHandler;
import com.alibaba.excel.write.metadata.holder.WriteSheetHolder;
import com.alibaba.excel.write.metadata.holder.WriteTableHolder;
import com.allinpay.core.util.ExcelStyleUtil;
import org.apache.poi.ss.usermodel.*;
/**
* 自定义行拦截器, 设置导出表格有序号列
*/
public class CustomRowWriteHandler implements RowWriteHandler {
/**
* 序号的样式,与其他列保持一样的样式
*/
private CellStyle firstCellStyle;
private static final String FIRST_CELL_NAME = "序号";
/**
* 列号
*/
private int count = 0;
@Override
public void beforeRowCreate(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, Integer integer, Integer integer1, Boolean aBoolean) {
}
@Override
public void afterRowCreate(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, Row row, Integer integer, Boolean aBoolean) {
// 每一行首列单元格
Cell cell = row.createCell(0);
if (firstCellStyle == null) {
Workbook workbook = writeSheetHolder.getSheet().getWorkbook();
firstCellStyle = ExcelStyleUtil.firstCellStyle(workbook);
}
//设置列宽 0列 10个字符宽度
writeSheetHolder.getSheet().setColumnWidth(0, 10 * 256);
if (row.getRowNum() == 0) {
cell.setCellValue(FIRST_CELL_NAME);
cell.setCellStyle(firstCellStyle);
return;
}
cell.setCellValue(++count);
}
@Override
public void afterRowDispose(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, Row row, Integer integer, Boolean aBoolean) {
}
}
CustomCellWeightWeightConfig类excel表列宽度自适应列数据
import com.alibaba.excel.enums.CellDataTypeEnum;
import com.alibaba.excel.metadata.CellData;
import com.alibaba.excel.metadata.Head;
import com.alibaba.excel.write.metadata.holder.WriteSheetHolder;
import com.alibaba.excel.write.style.column.AbstractColumnWidthStyleStrategy;
import org.apache.poi.ss.usermodel.Cell;
import org.springframework.util.CollectionUtils;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* 功能描述:根据列数据自适应宽度
**/
public class CustomCellWeightWeightConfig extends AbstractColumnWidthStyleStrategy {
// 可以根据这里的最大宽度,按自己需要进行调整,搭配单元格样式实现类中的,自动换行,效果更好
private static final int MAX_COLUMN_WIDTH = 50;
private Map<Integer, Map<Integer, Integer>> CACHE = new HashMap(8);
@Override
protected void setColumnWidth(WriteSheetHolder writeSheetHolder, List<CellData> cellDataList, Cell cell, Head head, Integer relativeRowIndex, Boolean isHead) {
boolean needSetWidth = isHead || !CollectionUtils.isEmpty(cellDataList);
if (needSetWidth) {
Map<Integer, Integer> maxColumnWidthMap = (Map)CACHE.get(writeSheetHolder.getSheetNo());
if (maxColumnWidthMap == null) {
maxColumnWidthMap = new HashMap(16);
CACHE.put(writeSheetHolder.getSheetNo(), maxColumnWidthMap);
}
Integer columnWidth = this.dataLength(cellDataList, cell, isHead);
if (columnWidth >= 0) {
if (columnWidth > MAX_COLUMN_WIDTH) {
columnWidth = MAX_COLUMN_WIDTH;
}
Integer maxColumnWidth = (Integer)((Map)maxColumnWidthMap).get(cell.getColumnIndex());
if (maxColumnWidth == null || columnWidth > maxColumnWidth) {
((Map)maxColumnWidthMap).put(cell.getColumnIndex(), columnWidth);
writeSheetHolder.getSheet().setColumnWidth(cell.getColumnIndex(), columnWidth * 256);
}
}
}
}
private Integer dataLength(List<CellData> cellDataList, Cell cell, Boolean isHead) {
if (isHead) {
return cell.getStringCellValue().getBytes().length;
} else {
CellData cellData = (CellData)cellDataList.get(0);
CellDataTypeEnum type = cellData.getType();
if (type == null) {
return -1;
} else {
switch(type) {
case STRING:
return cellData.getStringValue().getBytes().length;
case BOOLEAN:
return cellData.getBooleanValue().toString().getBytes().length;
case NUMBER:
return cellData.getNumberValue().toString().getBytes().length;
default:
return -1;
}
}
}
}
}
4、数据库查询数据生成txt文件上送到sftp服务器
实体类
import lombok.Data;
import lombok.NoArgsConstructor;
import com.alibaba.fastjson.annotation.JSONField;
@Data
@NoArgsConstructor
//@JSONField(ordinal = 0)转json时字段排序
public class PeopleTxt {
/**
* 序号
*/
@JSONField(ordinal = 0)
private String rowNo;
/**
* 姓名
*/
@JSONField(ordinal = 1)
private String name;
/**
* 年龄
*/
@JSONField(ordinal = 2)
private String age;
/**
* 联系电话
*/
@JSONField(ordinal = 3)
private String Phone;
/**
* 地址
*/
@JSONField(ordinal = 4)
private String address;
}
SQL
-- rank() 可以给每条数据产生一个顺序数
select rank() over(order by 字段) rowNo from 表名;
查询上送
private void uploadToSftp() {
String fileName = "people";
//sftp服务器的ip地址
String sftpIp = "127.0.0.1";
//sftp服务器的端口,一般默认22
String sftpPort = "22";
//sftp服务器的账号
String sftpAccount = "sftpAcc";
//sftp服务器的密码
String sftpPassWord = "sftpPw";
SftpConfig sftpConfig = new SftpConfig(sftpIp, sftpPort,
sftpAccount, sftpPassWord, "60000");
//查询所有people数据
List<PeopleExcelModal> peopleList = peopleMapper.queryAll();
//SerializerFeature.WriteMapNullValue空值时转为json字段也要存在
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
byteArrayOutputStream.write(jsonArryToStringBuilder(JSONArray.fromObject(JSON.toJSONString(peopleList,
SerializerFeature.WriteMapNullValue))).toString().getBytes());
//创建inputStream流
InputStream inputStream = new ByteArrayInputStream(byteArrayOutputStream.toByteArray());
sftpUtils.upload(sftpFileUrl, fileName + ".txt", inputStream, sftpConfig);
}
jsonArryToStringBuilder 将JsonArray转为StringBuilder
想要保住JsonArray中的字段数据顺序,要使用net.sf.json.JSONArray ,需要依赖
<dependency>
<groupId>net.sf.json-lib</groupId>
<artifactId>json-lib</artifactId>
<version>2.4</version>
<classifier>jdk15</classifier>
</dependency>
StringBuilder jsonArryToStringBuilder(JSONArray jsonArray) {
StringBuilder stringBuilder = new StringBuilder();
// 格式化浮点数据
NumberFormat formatter = NumberFormat.getNumberInstance();
// 设置最大小数位为10
formatter.setMaximumFractionDigits(10);
// 格式化日期数据
SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
for (int i = 0; i < jsonArray.size(); i++) {
JSONObject jsonObject = jsonArray.getJSONObject(i);
if (jsonObject.isEmpty()) {
break;
}
StringBuilder thisLine = new StringBuilder("");
for (Iterator<String> iterator = jsonObject.keySet().iterator(); iterator.hasNext(); ) {
// 当前字段
String key = iterator.next();
Object obj = jsonObject.get(key);
// 格式化数据
String field = "";
if (null != obj) {
if (obj.getClass() == String.class) {
// 如果是字符串
field = (String) obj;
} else if (obj.getClass() == Double.class || obj.getClass() == Float.class) {
// 格式化浮点数,使浮点数不以科学计数法输出
field = formatter.format(obj);
} else if (obj.getClass() == Integer.class || obj.getClass() == Long.class
|| obj.getClass() == Short.class || obj.getClass() == Byte.class) { // 如果是整形
field += obj;
} else if (obj.getClass() == Date.class) {
// 如果是日期类型
field = sdf.format(obj);
}
} else {
// null时给一个空格占位
field = " ";
}
// 拼接所有字段为一行数据, 逗号分隔
// 不是最后一个元素
if (iterator.hasNext()) {
thisLine.append(field).append(",");
} else {
// 是最后一个元素
thisLine.append(field);
}
}
stringBuilder.append(thisLine).append("\n");
}
return stringBuilder;
}
5、数据库查询数据生成csv文件到sftp文件
需要依赖
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>1.10.0</version>
</dependency>
实体类
import lombok.Data;
import lombok.NoArgsConstructor;
import com.alibaba.fastjson.annotation.JSONField;
@Data
@NoArgsConstructor
public class PeopleCsv {
/**
* 姓名
*/
private String name;
/**
* 年龄
*/
private String age;
/**
* 联系电话
*/
private String Phone;
/**
* 地址
*/
private String address;
}
查询上送
private void uploadToSftp() {
String fileName = "people";
//sftp服务器的ip地址
String sftpIp = "127.0.0.1";
//sftp服务器的端口,一般默认22
String sftpPort = "22";
//sftp服务器的账号
String sftpAccount = "sftpAcc";
//sftp服务器的密码
String sftpPassWord = "sftpPw";
SftpConfig sftpConfig = new SftpConfig(sftpIp, sftpPort,
sftpAccount, sftpPassWord, "60000");
//查询所有people数据
List<PeopleCsv> peopleList = peopleMapper.queryAll();
String[] tradeInfoHeader = {"姓名","年龄","联系电话","地址"};
//取保按实体类中的顺序和写的表头顺序相同
List<List<Object>> peopleLists = peopleList.stream().map(this::convert)
.collect(Collectors.toList());
//创建一个流,等待写入csv文件内容
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
//数据处理为csv文件
CSVUtils.exportCSVFile(byteArrayOutputStream, peopleList, "UTF-8", fileName, header);
SftpUtils sftpUtils = new SftpUtils(3, 6000);
//创建inputStream流
InputStream inputStream = new ByteArrayInputStream(byteArrayOutputStream.toByteArray());
// 上送sftp
sftpUtils.upload(sftpFileUrl, fileName, inputStream, sftpConfig);
}
convert类
public <T> List<Object> convert(T obj) {
List<Object> list = new ArrayList<>();
Class<?> clazz = obj.getClass();
for (Field field : clazz.getDeclaredFields()) {
field.setAccessible(true);
try {
list.add(field.get(obj));
} catch (IllegalAccessException e) {
e.printStackTrace();
}
}
return list;
}
CSVUtils类方法
/**
* 导出 csv 文件
* [@param] out 输出流
* [@param] iter 数据 我这里传 List<List<Object>> 类型
* [@param] charset 字符集编码
* [@param] header 表头
*/
public static void exportCSVFile(OutputStream out, Iterable<?> iter, String charset, String filePath, String... header ) {
try {
// 写入bom, 防止中文乱码
byte[] bytes = {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF};
out.write(bytes);
OutputStreamWriter osw = new OutputStreamWriter(out, charset);
CSVFormat csvFormat = CSVFormat.EXCEL.withHeader(header);
// 创建 CSVPrinter 对象,并设置列名和数据格式,生成到本地时可用
// CSVPrinter csvPrinter = new CSVPrinter(new FileWriter(filePath), CSVFormat.DEFAULT.withHeader(header));
CSVPrinter csvPrinter = new CSVPrinter(osw, csvFormat);
csvPrinter.printRecords(iter);
csvPrinter.flush();
csvPrinter.close();
} catch (IOException e) {
e.printStackTrace();
}
}
注意: 服务器空间满了的情况会导致生成失败