本文實例為大家分享了java線程池實現批量下載文件的具體代碼，供大家參考，具體內容如下

1 創建線程池

package com.cheng.webb.thread;

import java.util.concurrent.ArrayBlockingQueue;

import java.util.concurrent.ExecutorService;

import java.util.concurrent.ThreadFactory;

import java.util.concurrent.ThreadPoolExecutor;

import java.util.concurrent.TimeUnit;

import java.util.concurrent.atomic.AtomicInteger;

public class ThreadUtil {

/**

* 創建批量下載線程池

*

* @param threadSize 下載線程數

* @return ExecutorService

*/

public static ExecutorService buildDownloadBatchThreadPool(int threadSize) {

int keepAlive = 0;

String prefix = "download-batch";

ThreadFactory factory = ThreadUtil.buildThreadFactory(prefix);

return new ThreadPoolExecutor(threadSize,

threadSize,

keepAlive,

TimeUnit.SECONDS,

new ArrayBlockingQueue<>(threadSize),

factory);

}

/**

* 創建自定義線程工廠

*

* @param prefix 名稱前綴

* @return ThreadFactory

*/

public static ThreadFactory buildThreadFactory(String prefix) {

return new CustomThreadFactory(prefix);

}

/**

* 自定義線程工廠

*/

public static class CustomThreadFactory implements ThreadFactory {

private String threadNamePrefix;

private AtomicInteger counter = new AtomicInteger(1);

/**

* 自定義線程工廠

*

* @param threadNamePrefix 工廠名稱前綴

*/

CustomThreadFactory(String threadNamePrefix) {

this.threadNamePrefix = threadNamePrefix;

}

@Override

public Thread newThread(Runnable r) {

String threadName = threadNamePrefix + "-t" + counter.getAndIncrement();

return new Thread(r, threadName);

}

}

}

2 批量下載文件

package com.cheng.webb.thread;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

import java.io.File;

import java.io.FileOutputStream;

import java.io.InputStream;

import java.net.HttpURLConnection;

import java.net.URL;

import java.util.ArrayList;

import java.util.List;

import java.util.Map;

import java.util.concurrent.*;

/**

* 文件下載類

*

* @author shucheng

* @creation 2019年1月30日下午4:41:32

*/

public class DownloadUtil {

private static Logger logger = LoggerFactory.getLogger(DownloadUtil.class);

/**

* 下載線程數

*/

private static final int DOWNLOAD_THREAD_NUM = 14;

/**

* 下載線程池

*/

private static ExecutorService downloadExecutorService = ThreadUtil

.buildDownloadBatchThreadPool(DOWNLOAD_THREAD_NUM);

/**

* 文件下載

*

* @param fileUrl

* 文件url,如:https://img3.doubanio.com//view//photo//s_ratio_poster//public//p2369390663.webp

* @param path

* 存放路徑,如: /opt/img/douban/my.webp

*/

public static void download(String fileUrl, String path) {

// 判斷存儲文件夾是否已經存在或者創建成功

if (!createFolderIfNotExists(path)) {

logger.error("We can't create folder:{}", getFolder(path));

return;

}

InputStream in = null;

FileOutputStream out = null;

try {

URL url = new URL(fileUrl);

HttpURLConnection conn = (HttpURLConnection) url.openConnection();

conn.setRequestMethod("GET");

// 2s

conn.setConnectTimeout(10000);

in = conn.getInputStream();

out = new FileOutputStream(path);

int len;

byte[] arr = new byte[1024 * 1000];

while (-1 != (len = in.read(arr))) {

out.write(arr, 0, len);

}

out.flush();

conn.disconnect();

} catch (Exception e) {

logger.error("Fail to download: {} by {}", fileUrl, e.getMessage());

} finally {

try {

if (null != out) {

out.close();

}

if (null != in) {

in.close();

}

} catch (Exception e) {

// do nothing

}

}

}

/**

* 創建文件夾,如果文件夾已經存在或者創建成功返回true

*

* @param path

* 路徑

* @return boolean

*/

private static boolean createFolderIfNotExists(String path) {

String folderName = getFolder(path);

if (folderName.equals(path)) {

return true;

}

File folder = new File(getFolder(path));

if (!folder.exists()) {

synchronized (DownloadUtil.class) {

if (!folder.exists()) {

return folder.mkdirs();

}

}

}

return true;

}

/**

* 獲取文件夾

*

* @param path

* 文件路徑

* @return String

*/

private static String getFolder(String path) {

int index = path.lastIndexOf("/");

return -1 != index ? path.substring(0, index) : path;

}

/**

* 下載資源

*

* issue: 線程池創建過多

*

* 最大批量下載為5,請知悉

*

* @param resourceMap

* 資源map, key為資源下載url,value為資源存儲位置

*/

public static void batch(Map resourceMap) {

if (resourceMap == null || resourceMap.isEmpty()) {

return;

}

try {

List keys = new ArrayList<>(resourceMap.keySet());

int size = keys.size();

int pageNum = getPageNum(size);

for (int index = 0; index < pageNum; index++) {

int start = index * DOWNLOAD_THREAD_NUM;

int last = getLastNum(size, start + DOWNLOAD_THREAD_NUM);

final CountDownLatch latch = new CountDownLatch(last - start);

// 獲取列表子集

List urlList = keys.subList(start, last);

for (String url : urlList) {

// 提交任務

Runnable task = new DownloadWorker(latch, url, resourceMap.get(url));

downloadExecutorService.submit(task);

}

latch.await();

}

} catch (Exception e) {

logger.error("{}", e);

}

logger.info("Download resource map is all done");

}

/**

* 獲取最后一個元素

*

* @param size

* 列表長度

* @param index

* 下標

* @return int

*/

private static int getLastNum(int size, int index) {

return index > size ? size : index;

}

/**

* 獲取劃分頁面數量

*

* @param size

* 列表長度

* @return int

*/

private static int getPageNum(int size) {

int tmp = size / DOWNLOAD_THREAD_NUM;

return size % DOWNLOAD_THREAD_NUM == 0 ? tmp : tmp + 1;

}

/**

* 下載線程

*/

static class DownloadWorker implements Runnable {

private CountDownLatch latch;

private String url;

private String path;

DownloadWorker(CountDownLatch latch, String url, String path) {

this.latch = latch;

this.url = url;

this.path = path;

}

@Override

public void run() {

logger.debug("Start batch:[{}] into: [{}]", url, path);

DownloadUtil.download(url, path);

logger.debug("Download:[{}] into: [{}] is done", url, path);

latch.countDown();

}

}

}

3 測試批量下載文件

package com.cheng.webb.thread;

import java.util.HashMap;

import java.util.Map;

import org.junit.Test;

import com.alibaba.fastjson.JSON;

public class DownLoadTest {

String json = "{\r\n"

+ " \"http://www.xxx.com/111/123.mp4\":\"myFile/111/123.mp4\",\r\n"

+ " \"http://www.xxx.com/111/124.mp4\":\"myFile/111/124.mp4\",\r\n"

+ " \"http://www.xxx.com/111/125.mp4\":\"myFile/111/125.mp4\"\r\n"

+ "}";

@SuppressWarnings("unchecked")

@Test

public void test() {

Map map = new HashMap<>();

Map resMap = JSON.parseObject(json, map.getClass());

int times = 1;

for (int index = 0; index < times; index++) {

DownloadUtil.batch(resMap);

}

}

}

以上就是本文的全部內容，希望對大家的學習有所幫助，也希望大家多多支持腳本之家。

創作挑戰賽新人創作獎勵來咯，堅持創作打卡瓜分現金大獎

總結

以上是生活随笔為你收集整理的mysql 线程池 下载_java线程池实现批量下载文件的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。