日志挖掘——處理數據、計費統計

1、讀取附件中日志的內容，找出自己學號停車場中對應的進出車次數（in/out配對的記錄數，1條in、1條out，視為一個車次，本日志中in/out為一一對應，不存在缺失某條進或出記錄）

2、統計自己停車場車輛累計停放秒（即某個車牌，車輛駛出時間-車輛駛入時間=本此停放秒數，統計自己車場全部車牌的累計）

在日志挖掘實驗基礎上，針對車輛進出進行計費。

計費規則如下：

1、進場后在30分鐘內出場的車輛，免收費；

2、停放在2小時內（含2小時），收費10元；從第三小時起，每小時2元。

3、不封頂。

public class CountPark {static Map<String,Integer> profit=new HashMap<>();static Map<String,String> map=new HashMap<>();static long cnt=0,time=0;static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");static SimpleDateFormat sdf2 = new SimpleDateFormat("yyyy-MM");public static List<String> readTxt(String fileName){List<String> list=new ArrayList<>();try { // 防止文件建立或讀取失敗，用catch捕捉錯誤并打印，也可以throw/* 讀入TXT文件 */System.out.println(fileName);File filename = new File(fileName); // 要讀取以上路徑的input。txt文件InputStreamReader reader = new InputStreamReader(new FileInputStream(filename),"gbk"); // 建立一個輸入流對象readerBufferedReader br = new BufferedReader(reader); // 建立一個對象，它把文件內容轉成計算機能讀懂的語言String line = "";line = br.readLine();while (line != null) {String[] temp=line.split(",");if(temp[1].equals("2018250")){if(map.containsKey(temp[2])){long cur=sdf.parse(temp[0]).getTime();int cost=0;String month=sdf2.format(sdf.parse(temp[0]));long pre=sdf.parse(map.get(temp[2])).getTime();double curTime=((double) (cur-pre))/1000;//秒if(curTime>1800)//大于半小時{cost+=10;//收10元double hour= Math.ceil(curTime/(60*60));//小時if(hour>2)//兩小時以上cost+=(hour-2)*2;}profit.put(month,profit.getOrDefault(month,0)+cost);//記錄map.remove(temp[2]);}elsemap.put(temp[2],temp[0]);cnt++;}line = br.readLine();}} catch (Exception e) {e.printStackTrace();}return list;}public static void main(String[] args) {long c=System.currentTimeMillis();List<String> list=readTxt("cars2.txt");System.out.println("運行時間（秒）:"+(double)(System.currentTimeMillis()-c)/1000+"秒");for(String s:profit.keySet())System.out.println(s+":"+profit.get(s));} }

日志挖掘——并行改造

1、讀取文件并行化（數據分割）

具體做法：針對之前一個大日志文件，根據時間（例如按照日期）進行切割，切割為多個小文件。讀取時，每天的數據分別讀取

2、處理并行化（任務分割）

具體做法：針對每個日期，分別進行各種掃描和統計；對于跨日期的數據，存放在公用地方，到時再配對處理

public class CountPark2 {static Map<String,ArrayList<String>> profit=new HashMap<>();static Map<String,String> map=new HashMap<>();static long cnt=0,time=0;static SimpleDateFormat formatter = new SimpleDateFormat("HH:mm:ss.SSS");static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");static SimpleDateFormat sdf2 = new SimpleDateFormat("yyyy-MM");public static void readTxt(String fileName){List<String> list=new ArrayList<>();try { // 防止文件建立或讀取失敗，用catch捕捉錯誤并打印，也可以throw/* 讀入TXT文件 */File filename = new File(fileName); // 要讀取以上路徑的input。txt文件InputStreamReader reader = new InputStreamReader(new FileInputStream(filename),"gbk"); // 建立一個輸入流對象readerBufferedReader br = new BufferedReader(reader); // 建立一個對象，它把文件內容轉成計算機能讀懂的語言String line = "";line = br.readLine();while (line != null) {String[] temp=line.split(",");if(temp[1].equals("2018250")){long cur=sdf.parse(temp[0]).getTime(); String month=sdf2.format(sdf.parse(temp[0]));if(!profit.containsKey(month)) profit.put(month,new ArrayList<>());profit.get(month).add(line);}line = br.readLine();} }catch (Exception e) {e.printStackTrace();}}public static void writeTxt(String content,String filename){try {File writename = new File("src/logminning/reso/"+filename); // 相對路徑，如果沒有則要建立一個新的output.txt文件writename.createNewFile(); // 創建新文件BufferedWriter out = new BufferedWriter(new FileWriter(writename));out.write(content); // \r\n即為換行out.flush(); // 把緩存區內容壓入文件out.close(); // 最后記得關閉文件} catch (Exception e) {e.printStackTrace();}}public void handle(){readTxt("cars2.txt");for(String c:profit.keySet()){StringBuilder stringBuilder=new StringBuilder();for(String t:profit.get(c))stringBuilder.append(t).append("\r\n");writeTxt(stringBuilder.toString(),c);}}public static void main(String[] args) {ConcurrentHashMap<String, PriorityBlockingQueue<Long>> concurrentHashMap=new ConcurrentHashMap<>();BlockingQueue<String> blockingQueue=new LinkedBlockingQueue<>();ExecutorService executorService= Executors.newFixedThreadPool(8);CountDownLatch countDownLatch=new CountDownLatch(6);System.out.println("Start ReadFile:"+formatter.format(new Date()));for(int i=1;i<=6;i++){executorService.execute(new ReadFileThread(concurrentHashMap,"src/logminning/reso/2020-0"+String.valueOf(i)+".txt",countDownLatch));}try {countDownLatch.await();} catch (InterruptedException e) {e.printStackTrace();}System.out.println("End ReadFile:"+formatter.format(new Date())); /* int used=0;for(String c:concurrentHashMap.keySet()){if(concurrentHashMap.get(c).size()==2) continue;System.out.println(c+':'+concurrentHashMap.get(c).size());used+=concurrentHashMap.get(c).size();}System.out.println(used);*/Res res=new Res();countDownLatch=new CountDownLatch(6);for(int i=0;i<26;i+=5){executorService.execute(new handler(concurrentHashMap,countDownLatch,i,res));}try {countDownLatch.await();} catch (InterruptedException e) {e.printStackTrace();}System.out.println("End Process:"+formatter.format(new Date()));System.out.println(res.getTar()+":"+res.getUsed());executorService.shutdown();} } public class handler implements Runnable{CountDownLatch countDownLatch;ConcurrentHashMap<String, PriorityBlockingQueue<Long>> concurrentHashMap;int s;Res res;public handler(ConcurrentHashMap<String, PriorityBlockingQueue<Long>> concurrentHashMap,CountDownLatch countDownLatch,int s,Res res) {this.s=s;this.res=res;this.countDownLatch=countDownLatch;this.concurrentHashMap = concurrentHashMap;}@Overridepublic void run() {try {for(String c:concurrentHashMap.keySet() ){int cur=c.charAt(1)-'A';if(cur>=s&&cur<s+5){PriorityBlockingQueue<Long> priorityQueue=concurrentHashMap.get(c);while (priorityQueue.size()>0){long c1=priorityQueue.take(),c2=priorityQueue.take();res.add((c2-c1)/1000);}}}} catch (Exception e) {e.printStackTrace();}countDownLatch.countDown();} } public class ReadFileThread implements Runnable {private ConcurrentHashMap<String, PriorityBlockingQueue<Long>> concurrentHashMap;private String fileName;CountDownLatch countDownLatch;SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");public ReadFileThread(ConcurrentHashMap<String, PriorityBlockingQueue<Long>> concurrentHashMap, String filename, CountDownLatch countDownLatch) {this.concurrentHashMap = concurrentHashMap;this.fileName = filename;this.countDownLatch=countDownLatch;}@Overridepublic void run() {try { // 防止文件建立或讀取失敗，用catch捕捉錯誤并打印，也可以throw/* 讀入TXT文件 */File filename = new File(fileName); // 要讀取以上路徑的input。txt文件InputStreamReader reader = new InputStreamReader(new FileInputStream(filename),"UTF-8"); // 建立一個輸入流對象readerBufferedReader br = new BufferedReader(reader); // 建立一個對象，它把文件內容轉成計算機能讀懂的語言String line = "";line = br.readLine();while (line != null) {String[] temp=line.split(",");if(!concurrentHashMap.containsKey(temp[2]))concurrentHashMap.put(temp[2],new PriorityBlockingQueue<>());synchronized (this){long cur=sdf.parse(temp[0]).getTime(); concurrentHashMap.get(temp[2]).add(cur);}line = br.readLine();}} catch (Exception e) {e.printStackTrace();}countDownLatch.countDown();} } public class Res {private long tar = 0;private int used=0;public long getTar() {return tar;}public int getUsed() {return used;}public synchronized void add(long c){tar+=c;used++;} }

總結

以上是生活随笔為你收集整理的分布与并行计算—日志挖掘（Java）的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。