diff --git a/collection/pom.xml b/collection/pom.xml new file mode 100644 index 0000000..6d715c3 --- /dev/null +++ b/collection/pom.xml @@ -0,0 +1,97 @@ + + + + + parent + ru.bvn13.imdbspider + 0.0.1 + + 4.0.0 + + imdb-spider-collection + + IMDB-SPIDER :: COLLECTION + + + UTF-8 + 11 + 11 + + + + + junit + junit + 4.11 + test + + + ru.bvn13.imdbspider + imdb-spider-core + 0.0.1 + compile + + + + + args4j + args4j + 2.33 + + + + + org.apache.commons + commons-lang3 + 3.8.1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/collection/src/main/java/module-info.java b/collection/src/main/java/module-info.java new file mode 100644 index 0000000..0afe83c --- /dev/null +++ b/collection/src/main/java/module-info.java @@ -0,0 +1,10 @@ +module imdb.spider.collection { + + opens ru.bvn13.imdbspider.collection; + + requires imdb.spider.core; + + requires args4j; + requires org.apache.commons.lang3; + +} \ No newline at end of file diff --git a/collection/src/main/java/ru/bvn13/imdbspider/collection/Runner.java b/collection/src/main/java/ru/bvn13/imdbspider/collection/Runner.java new file mode 100644 index 0000000..9ece1e1 --- /dev/null +++ b/collection/src/main/java/ru/bvn13/imdbspider/collection/Runner.java @@ -0,0 +1,199 @@ +package ru.bvn13.imdbspider.collection; + +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; +import org.kohsuke.args4j.CmdLineException; +import org.kohsuke.args4j.CmdLineParser; +import org.kohsuke.args4j.Option; +import ru.bvn13.imdbspider.collection.composer.CollectionComposer; +import ru.bvn13.imdbspider.collection.exceptions.CollectionComposerException; + +/** + * Hello world! + * + */ +public class Runner { + + public static final class Settings { + + @Option(name = "--library-path", aliases = {"-l"}, usage = "Library path", required = true) + private String libraryPath; + + @Option(name = "--output-path", aliases = {"-o"}, usage = "Output path", required = true) + private String outputPath; + + @Option(name = "--recursive", aliases = {"-r"}, usage = "Recursive") + private boolean isRecursive; + + @Option(name = "--collection-template", aliases = {"-ct"}, usage = "HTML as Collection template") + private String collectionTemplatePath; + + @Option(name = "--movie-template", aliases = {"-mt"}, usage = "HTML as Movie template") + private String movieTemplatePath; + + @Option(name = "--row-template", aliases = {"-rt"}, usage = "HTML as row template") + private String rowTemplatePath; + + @Option(name = "--movies-per-row", aliases = {"-mr"}, usage = "Number, count of movies into one row (default = 3)") + private int moviesPerRow; + + @Option(name = "--collection-identifier", aliases = {"-ci"}, usage = "Substring in Collection template to be replaced with composed Collection block (default = '{{collection}}')") + private String collectionIdentifier; + + @Option(name = "--row-identifier", aliases = {"-ri"}, usage = "Substring in Row template to be replaced with composed Row block (default = '{{row]]')") + private String rowIdentigier; + + @Option(name = "--movie-title-identifier", aliases = {"-mti"}, usage = "Substring in Movie template to be replaced with movie title (default '{{title}}')") + private String movieTitleIdentifier; + + @Option(name = "--movie-poster-identifier", aliases = {"-mpi"}, usage = "Substring in Movie template to be replaced with movie poster (default '{{poster}}')") + private String moviePosterIdentifier; + + @Option(name = "--movie-filename-identifier", aliases = {"-mfi"}, usage = "Substring in Movie template to be replaced with movie filename (default '{{filename}}')") + private String movieFilenameIdentifier; + + @Option(name = "--categorized", aliases = {"-c"}, usage = "Use first level directory into Library path as category") + private boolean isCategorized; + + @Option(name = "--encoding", aliases = {"-e"}, usage = "Encoding (default = utf-8)") + private String encoding; + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this, ToStringStyle.MULTI_LINE_STYLE); + } + + //#region _GETTERS_ + + public String getLibraryPath() { + return libraryPath; + } + + public String getOutputPath() { + return outputPath; + } + + public boolean isRecursive() { + return isRecursive; + } + + public String getCollectionTemplatePath() { + return collectionTemplatePath; + } + + public String getMovieTemplatePath() { + return movieTemplatePath; + } + + public String getCollectionIdentifier() { + return collectionIdentifier; + } + + public boolean isCategorized() { + return isCategorized; + } + + public String getEncoding() { + return encoding; + } + + public String getMovieTitleIdentifier() { + return movieTitleIdentifier; + } + + public String getMoviePosterIdentifier() { + return moviePosterIdentifier; + } + + public String getRowTemplatePath() { + return rowTemplatePath; + } + + public int getMoviesPerRow() { + return moviesPerRow; + } + + public String getRowIdentigier() { + return rowIdentigier; + } + + //#endregion + } + + private void fillUpWithDefaults(Settings settings) { + + if (settings.encoding == null || settings.encoding.trim().isEmpty()) { + settings.encoding = "utf-8"; + } + + if (settings.collectionTemplatePath == null || settings.collectionTemplatePath.isEmpty()) { + settings.collectionTemplatePath = getClass().getResource("/templates/collection.html").getPath(); + } + + if (settings.movieTemplatePath == null || settings.movieTemplatePath.isEmpty()) { + settings.movieTemplatePath = getClass().getResource("/templates/movie.html").getPath(); + } + + if (settings.rowTemplatePath == null || settings.rowTemplatePath.isEmpty()) { + settings.rowTemplatePath = getClass().getResource("/templates/row.html").getPath(); + } + + if (settings.collectionIdentifier == null || settings.collectionIdentifier.trim().isEmpty()) { + settings.collectionIdentifier = "{{collection}}"; + } + + if (settings.rowIdentigier == null || settings.rowIdentigier.isEmpty()) { + settings.rowIdentigier = "{{row}}"; + } + + if (settings.movieTitleIdentifier == null || settings.movieTitleIdentifier.isEmpty()) { + settings.movieTitleIdentifier = "{{title}}"; + } + + if (settings.moviePosterIdentifier == null || settings.moviePosterIdentifier.isEmpty()) { + settings.moviePosterIdentifier = "{{poster}}"; + } + + if (settings.movieFilenameIdentifier == null || settings.movieFilenameIdentifier.isEmpty()) { + settings.movieFilenameIdentifier = "{{filename}}"; + } + + if (settings.moviesPerRow <= 0) { + settings.moviesPerRow = 3; + } + + } + + + private void start(String[] args) throws CollectionComposerException { + Settings settings = new Settings(); + CmdLineParser parser = new CmdLineParser(settings); + + try { + parser.parseArgument(args); + System.out.println("settings = " + settings); + } catch (CmdLineException e) { + System.err.println("e = " + e.toString()); + parser.printUsage(System.out); + return; + } + + fillUpWithDefaults(settings); + + CollectionComposer composer = new CollectionComposer(); + + composer.compose(settings); + } + + + public static void main(String[] args) throws CollectionComposerException { + + Runner runner = new Runner(); + runner.start(args); + + + } + + + +} diff --git a/collection/src/main/java/ru/bvn13/imdbspider/collection/composer/CollectionComposer.java b/collection/src/main/java/ru/bvn13/imdbspider/collection/composer/CollectionComposer.java new file mode 100644 index 0000000..af04448 --- /dev/null +++ b/collection/src/main/java/ru/bvn13/imdbspider/collection/composer/CollectionComposer.java @@ -0,0 +1,209 @@ +package ru.bvn13.imdbspider.collection.composer; + +import ru.bvn13.imdbspider.collection.Runner; +import ru.bvn13.imdbspider.collection.exceptions.CollectionComposerException; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +/** + * Created by bvn13 on 27.01.2019. + */ +public class CollectionComposer { + + private static final class ProcessFile extends SimpleFileVisitor { + private FileTreeElement treeElement; + private List filters = new ArrayList<>(); + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + boolean isApropriate = false; + for (String filter : filters) { + if (file.getFileName().toString().endsWith(filter) || file.getFileName().toString().matches(filter)) { + isApropriate = true; + break; + } + } + if (isApropriate) { + //System.out.println("Processing file:" + file); + FileTreeElement treeFile = new FileTreeElement(); + treeFile.setParent(treeElement); + treeFile.setPath(file); + treeFile.setFullPath(file.toString()); + treeFile.setType(FileTreeElement.TYPE.FILE); + treeElement.getNestedElements().add(treeFile); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + //System.out.println("Processing directory:" + dir); + if (treeElement == null) { + treeElement = new FileTreeElement(); + } else if (!treeElement.getFullPath().equals(dir.toString())) { + FileTreeElement subTreeElement = new FileTreeElement(); + subTreeElement.setParent(treeElement); + treeElement.getNestedElements().add(subTreeElement); + treeElement = subTreeElement; + } + treeElement.setPath(dir); + treeElement.setFullPath(dir.toString()); + treeElement.setType(FileTreeElement.TYPE.DIRECTORY); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path path, IOException exc) throws IOException { + return FileVisitResult.SKIP_SUBTREE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + FileTreeElement current = treeElement; + treeElement = treeElement.getParent(); + if (current.getNestedElements().size() == 0) { + treeElement.getNestedElements().remove(current); + current = null; + } + return FileVisitResult.CONTINUE; + } + + public List getFilters() { + return filters; + } + } + + private FileTreeElement tree; + private String collectionTemplate; + private String movieTemplate; + private String rowTemplate; + + private ExecutorService executorService; + + public void compose(Runner.Settings settings) throws CollectionComposerException { + + findAllVideoFiles(settings); + + Charset charset = Charset.forName(settings.getEncoding()); + + try { + collectionTemplate = readFile(settings.getCollectionTemplatePath(), charset); + movieTemplate = readFile(settings.getMovieTemplatePath(), charset); + rowTemplate = readFile(settings.getRowTemplatePath(), charset); + } catch (IOException e) { + throw new CollectionComposerException("Error reading file", e); + } + + AtomicInteger threadsCount = new AtomicInteger(0); + AtomicInteger completedCount = new AtomicInteger(0); + + Object object = new Object(); + + executorService = Executors.newCachedThreadPool(); + + startMovieRetriever(tree, threadsCount, completedCount, fname -> { + synchronized (object) { + System.out.print(String.format("Completed: [%d/%d]\r", completedCount.get(), threadsCount.get())); + } + }); + + executorService.shutdown(); + + try { + executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + throw new CollectionComposerException("Could not retrieve movie data", e); + } + + System.out.println("\n"); + + + + } + + private void findAllVideoFiles(Runner.Settings settings) { + tree = new FileTreeElement(); + tree.setFullPath(settings.getLibraryPath()); + + ProcessFile fileProcessor = new ProcessFile(); + fileProcessor.treeElement = tree; + + fileProcessor.getFilters().add(".+\\.3g2$"); + fileProcessor.getFilters().add(".+\\.3gp$"); + fileProcessor.getFilters().add(".+\\.3gp2$"); + fileProcessor.getFilters().add(".+\\.3gpp$"); + fileProcessor.getFilters().add(".+\\.3gpp2$"); + fileProcessor.getFilters().add(".+\\.asf$"); + fileProcessor.getFilters().add(".+\\.asx$"); + fileProcessor.getFilters().add(".+\\.avi$"); + fileProcessor.getFilters().add(".+\\.bin$"); + fileProcessor.getFilters().add(".+\\.dat$"); + fileProcessor.getFilters().add(".+\\.drv$"); + fileProcessor.getFilters().add(".+\\.f4v$"); + fileProcessor.getFilters().add(".+\\.flv$"); + fileProcessor.getFilters().add(".+\\.gtp$"); + fileProcessor.getFilters().add(".+\\.h264$"); + fileProcessor.getFilters().add(".+\\.m4v$"); + fileProcessor.getFilters().add(".+\\.mkv$"); + fileProcessor.getFilters().add(".+\\.mod$"); + fileProcessor.getFilters().add(".+\\.moov$"); + fileProcessor.getFilters().add(".+\\.mov$"); + fileProcessor.getFilters().add(".+\\.mp4$"); + fileProcessor.getFilters().add(".+\\.mpeg$"); + fileProcessor.getFilters().add(".+\\.mpg$"); + fileProcessor.getFilters().add(".+\\.mts$"); + fileProcessor.getFilters().add(".+\\.rm$"); + fileProcessor.getFilters().add(".+\\.rmvb$"); + fileProcessor.getFilters().add(".+\\.spl$"); + fileProcessor.getFilters().add(".+\\.srt$"); + fileProcessor.getFilters().add(".+\\.stl$"); + fileProcessor.getFilters().add(".+\\.swf$"); + fileProcessor.getFilters().add(".+\\.ts$"); + fileProcessor.getFilters().add(".+\\.vcd$"); + fileProcessor.getFilters().add(".+\\.vid$"); + fileProcessor.getFilters().add(".+\\.vid$"); + fileProcessor.getFilters().add(".+\\.vid$"); + fileProcessor.getFilters().add(".+\\.vob$"); + fileProcessor.getFilters().add(".+\\.webm$"); + fileProcessor.getFilters().add(".+\\.wm$"); + fileProcessor.getFilters().add(".+\\.wmv$"); + fileProcessor.getFilters().add(".+\\.yuv$"); + + try { + Files.walkFileTree(Paths.get(settings.getLibraryPath()), fileProcessor); + } catch (IOException e) { + e.printStackTrace(); + } + + tree.print(); + } + + private static String readFile(String path, Charset encoding) throws IOException { + byte[] encoded = Files.readAllBytes(Paths.get(path)); + return new String(encoded, encoding); + } + + private void startMovieRetriever(FileTreeElement treeElement, AtomicInteger threadsCount, AtomicInteger completedCount, Consumer callback) { + + if (treeElement.isFile()) { + threadsCount.incrementAndGet(); + executorService.submit(new MovieRetriever(treeElement, completedCount, callback)); + } else if (treeElement.isDirectory()) { + for (FileTreeElement nested : treeElement.getNestedElements()) { + startMovieRetriever(nested, threadsCount, completedCount, callback); + } + } + + } + +} diff --git a/collection/src/main/java/ru/bvn13/imdbspider/collection/composer/FileTreeElement.java b/collection/src/main/java/ru/bvn13/imdbspider/collection/composer/FileTreeElement.java new file mode 100644 index 0000000..d0f0002 --- /dev/null +++ b/collection/src/main/java/ru/bvn13/imdbspider/collection/composer/FileTreeElement.java @@ -0,0 +1,96 @@ +package ru.bvn13.imdbspider.collection.composer; + +import ru.bvn13.imdbspider.imdb.Movie; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/** + * Created by bvn13 on 28.01.2019. + */ +public class FileTreeElement { + enum TYPE { + UNKNOWN, FILE, DIRECTORY + } + + private TYPE type = TYPE.UNKNOWN; + + private Path path; + private String fullPath; + private FileTreeElement parent; + private List nestedElements = new ArrayList<>(); + private int depth = 0; + + private Movie movie; + + //#region _GETTERS_SETTERS_ + + public Path getPath() { + return path; + } + + public void setPath(Path path) { + this.path = path; + } + + public String getFullPath() { + return fullPath; + } + + public void setFullPath(String fullPath) { + this.fullPath = fullPath; + } + + public FileTreeElement getParent() { + return parent; + } + + public void setParent(FileTreeElement parent) { + this.parent = parent; + this.depth = parent.depth + 1; + } + + public List getNestedElements() { + return nestedElements; + } + + public void setType(TYPE type) { + this.type = type; + } + + public boolean isDirectory() { + return type.equals(TYPE.DIRECTORY); + } + + public boolean isFile() { + return type.equals(TYPE.FILE); + } + + public int getDepth() { + return depth; + } + + public Movie getMovie() { + return movie; + } + + public void setMovie(Movie movie) { + this.movie = movie; + } + + //#endregion + + public void print() { + StringBuilder value = new StringBuilder(); + for (int i=0; i "); + value.append(fullPath); + System.out.println(value); + for (FileTreeElement nested : nestedElements) { + nested.print(); + } + } +} \ No newline at end of file diff --git a/collection/src/main/java/ru/bvn13/imdbspider/collection/composer/MovieRetriever.java b/collection/src/main/java/ru/bvn13/imdbspider/collection/composer/MovieRetriever.java new file mode 100644 index 0000000..c25ce32 --- /dev/null +++ b/collection/src/main/java/ru/bvn13/imdbspider/collection/composer/MovieRetriever.java @@ -0,0 +1,54 @@ +package ru.bvn13.imdbspider.collection.composer; + +import ru.bvn13.imdbspider.ImdbSpider; +import ru.bvn13.imdbspider.exceptions.ImdbSpiderException; +import ru.bvn13.imdbspider.imdb.MovieList; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +/** + * Created by bvn13 on 28.01.2019. + */ +public class MovieRetriever implements Runnable { + + private FileTreeElement treeElement; + private AtomicInteger completedCount; + private Consumer callback; + + public MovieRetriever(FileTreeElement treeElement, AtomicInteger completedCount, Consumer callback) { + this.treeElement = treeElement; + this.completedCount = completedCount; + this.callback = callback; + } + + private String prepareFileName(String roughFileName) { + String filename = ""; + + return roughFileName; + } + + + @Override + public void run() { + + String roughFilename = treeElement.getPath().getFileName().toString(); + String filename = prepareFileName(roughFilename); + + ImdbSpider spider = ImdbSpider.withApi_1_0(); + try { + MovieList result = spider.searchMovieByTitle(filename); + + + + } catch (ImdbSpiderException e) { + e.printStackTrace(); + } + + completedCount.incrementAndGet(); + + callback.accept(roughFilename); + } + + +} diff --git a/collection/src/main/java/ru/bvn13/imdbspider/collection/exceptions/CollectionComposerException.java b/collection/src/main/java/ru/bvn13/imdbspider/collection/exceptions/CollectionComposerException.java new file mode 100644 index 0000000..286de49 --- /dev/null +++ b/collection/src/main/java/ru/bvn13/imdbspider/collection/exceptions/CollectionComposerException.java @@ -0,0 +1,25 @@ +package ru.bvn13.imdbspider.collection.exceptions; + +/** + * Created by bvn13 on 28.01.2019. + */ +public class CollectionComposerException extends Exception { + public CollectionComposerException() { + } + + public CollectionComposerException(String message) { + super(message); + } + + public CollectionComposerException(String message, Throwable cause) { + super(message, cause); + } + + public CollectionComposerException(Throwable cause) { + super(cause); + } + + public CollectionComposerException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/collection/src/main/resources/templates/collection.html b/collection/src/main/resources/templates/collection.html new file mode 100644 index 0000000..d9f7ebe --- /dev/null +++ b/collection/src/main/resources/templates/collection.html @@ -0,0 +1,28 @@ + + + + + bvn13's Collection + + + + + + + + + + + + + + + +
+ +
{{collection}}
+ +
+ + + \ No newline at end of file diff --git a/collection/src/main/resources/templates/movie.html b/collection/src/main/resources/templates/movie.html new file mode 100644 index 0000000..23fb736 --- /dev/null +++ b/collection/src/main/resources/templates/movie.html @@ -0,0 +1,7 @@ +
+ +

{{title}}

+

{{filename}}

+
+ +
\ No newline at end of file diff --git a/collection/src/main/resources/templates/row.html b/collection/src/main/resources/templates/row.html new file mode 100644 index 0000000..fc70fc4 --- /dev/null +++ b/collection/src/main/resources/templates/row.html @@ -0,0 +1,3 @@ +
+ {{row}} +
\ No newline at end of file diff --git a/collection/src/test/java/ru/bvn13/imdbspider/collection/RunnerTest.java b/collection/src/test/java/ru/bvn13/imdbspider/collection/RunnerTest.java new file mode 100644 index 0000000..6080025 --- /dev/null +++ b/collection/src/test/java/ru/bvn13/imdbspider/collection/RunnerTest.java @@ -0,0 +1,20 @@ +package ru.bvn13.imdbspider.collection; + +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** + * Unit test for simple App. + */ +public class RunnerTest +{ + /** + * Rigorous Test :-) + */ + @Test + public void shouldAnswerWithTrue() + { + assertTrue( true ); + } +} diff --git a/pom.xml b/pom.xml index fc93ce8..1e5d273 100644 --- a/pom.xml +++ b/pom.xml @@ -10,6 +10,7 @@ core runner + collection IMDB-SPIDER :: PARENT