package ru.bvn13.imdbspider.spider.tasker; import ru.bvn13.imdbspider.exceptions.extractor.HtmlExtractorException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.*; /** * @author boyko_vn at 09.01.2019 */ public class Manager { private boolean isDebug; private Map httpRequestHeaders = new HashMap<>(); public boolean isDebug() { return isDebug; } public void setDebug(boolean debug) { isDebug = debug; } public void addHttpRequestHeader(String key, String value) { this.httpRequestHeaders.put(key, value); } public void processTasks(List allTasks) { Map> groupedTasks = new ConcurrentHashMap<>(allTasks.size()); for (Task task : allTasks) { List filteredTasks = null; if (groupedTasks.keySet().contains(task.getUrl())) { filteredTasks = groupedTasks.get(task.getUrl()); } else { filteredTasks = new ArrayList<>(); groupedTasks.put(task.getUrl(), filteredTasks); } filteredTasks.add(task); } groupedTasks.entrySet().parallelStream().forEach(stringListEntry -> { Worker w = new Worker(stringListEntry.getKey(), stringListEntry.getValue()); w.setDebug(isDebug); try { w.run(httpRequestHeaders); } catch (HtmlExtractorException e) { e.printStackTrace(); } }); List nextTasks = new ArrayList<>(); for (Task task : allTasks) { if (task.hasNextTasks()) { nextTasks.addAll(task.getNestedTasks()); } } if (!nextTasks.isEmpty()) { processTasks(nextTasks); } } }