bvn13
/
imdb-spider
mirror of https://github.com/bvn13/imdb-spider.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

working on Collection composer

develop
Vyacheslav N. Boyko 2019-01-28 01:08:09 +03:00
parent e568eeed3d
commit 60cb5ed534
12 changed files with 749 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
collection/pom.xml 100644
View File

@ -0,0 +1,97 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>parent</artifactId>
<groupId>ru.bvn13.imdbspider</groupId>
<version>0.0.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>imdb-spider-collection</artifactId>
<name>IMDB-SPIDER :: COLLECTION</name>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ru.bvn13.imdbspider</groupId>
<artifactId>imdb-spider-core</artifactId>
<version>0.0.1</version>
<scope>compile</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/args4j/args4j -->
<dependency>
<groupId>args4j</groupId>
<artifactId>args4j</artifactId>
<version>2.33</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.8.1</version>
</dependency>
</dependencies>
<build>
<pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
<!--<plugins>-->
<!--&lt;!&ndash; clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle &ndash;&gt;-->
<!--<plugin>-->
<!--<artifactId>maven-clean-plugin</artifactId>-->
<!--<version>3.1.0</version>-->
<!--</plugin>-->
<!--&lt;!&ndash; default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging &ndash;&gt;-->
<!--<plugin>-->
<!--<artifactId>maven-resources-plugin</artifactId>-->
<!--<version>3.0.2</version>-->
<!--</plugin>-->
<!--<plugin>-->
<!--<artifactId>maven-compiler-plugin</artifactId>-->
<!--<version>3.8.0</version>-->
<!--</plugin>-->
<!--<plugin>-->
<!--<artifactId>maven-surefire-plugin</artifactId>-->
<!--<version>2.22.1</version>-->
<!--</plugin>-->
<!--<plugin>-->
<!--<artifactId>maven-jar-plugin</artifactId>-->
<!--<version>3.0.2</version>-->
<!--</plugin>-->
<!--<plugin>-->
<!--<artifactId>maven-install-plugin</artifactId>-->
<!--<version>2.5.2</version>-->
<!--</plugin>-->
<!--<plugin>-->
<!--<artifactId>maven-deploy-plugin</artifactId>-->
<!--<version>2.8.2</version>-->
<!--</plugin>-->
<!--&lt;!&ndash; site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle &ndash;&gt;-->
<!--<plugin>-->
<!--<artifactId>maven-site-plugin</artifactId>-->
<!--<version>3.7.1</version>-->
<!--</plugin>-->
<!--<plugin>-->
<!--<artifactId>maven-project-info-reports-plugin</artifactId>-->
<!--<version>3.0.0</version>-->
<!--</plugin>-->
<!--</plugins>-->
</pluginManagement>
</build>
</project>

10
collection/src/main/java/module-info.java 100644
View File

@ -0,0 +1,10 @@
module imdb.spider.collection {
opens ru.bvn13.imdbspider.collection;
requires imdb.spider.core;
requires args4j;
requires org.apache.commons.lang3;
}

199
collection/src/main/java/ru/bvn13/imdbspider/collection/Runner.java 100644
View File

@ -0,0 +1,199 @@
package ru.bvn13.imdbspider.collection;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
import ru.bvn13.imdbspider.collection.composer.CollectionComposer;
import ru.bvn13.imdbspider.collection.exceptions.CollectionComposerException;
/**
* Hello world!
*
*/
public class Runner {
public static final class Settings {
@Option(name = "--library-path", aliases = {"-l"}, usage = "Library path", required = true)
private String libraryPath;
@Option(name = "--output-path", aliases = {"-o"}, usage = "Output path", required = true)
private String outputPath;
@Option(name = "--recursive", aliases = {"-r"}, usage = "Recursive")
private boolean isRecursive;
@Option(name = "--collection-template", aliases = {"-ct"}, usage = "HTML as Collection template")
private String collectionTemplatePath;
@Option(name = "--movie-template", aliases = {"-mt"}, usage = "HTML as Movie template")
private String movieTemplatePath;
@Option(name = "--row-template", aliases = {"-rt"}, usage = "HTML as row template")
private String rowTemplatePath;
@Option(name = "--movies-per-row", aliases = {"-mr"}, usage = "Number, count of movies into one row (default = 3)")
private int moviesPerRow;
@Option(name = "--collection-identifier", aliases = {"-ci"}, usage = "Substring in Collection template to be replaced with composed Collection block (default = '{{collection}}')")
private String collectionIdentifier;
@Option(name = "--row-identifier", aliases = {"-ri"}, usage = "Substring in Row template to be replaced with composed Row block (default = '{{row]]')")
private String rowIdentigier;
@Option(name = "--movie-title-identifier", aliases = {"-mti"}, usage = "Substring in Movie template to be replaced with movie title (default '{{title}}')")
private String movieTitleIdentifier;
@Option(name = "--movie-poster-identifier", aliases = {"-mpi"}, usage = "Substring in Movie template to be replaced with movie poster (default '{{poster}}')")
private String moviePosterIdentifier;
@Option(name = "--movie-filename-identifier", aliases = {"-mfi"}, usage = "Substring in Movie template to be replaced with movie filename (default '{{filename}}')")
private String movieFilenameIdentifier;
@Option(name = "--categorized", aliases = {"-c"}, usage = "Use first level directory into Library path as category")
private boolean isCategorized;
@Option(name = "--encoding", aliases = {"-e"}, usage = "Encoding (default = utf-8)")
private String encoding;
@Override
public String toString() {
return ToStringBuilder.reflectionToString(this, ToStringStyle.MULTI_LINE_STYLE);
}
//#region _GETTERS_
public String getLibraryPath() {
return libraryPath;
}
public String getOutputPath() {
return outputPath;
}
public boolean isRecursive() {
return isRecursive;
}
public String getCollectionTemplatePath() {
return collectionTemplatePath;
}
public String getMovieTemplatePath() {
return movieTemplatePath;
}
public String getCollectionIdentifier() {
return collectionIdentifier;
}
public boolean isCategorized() {
return isCategorized;
}
public String getEncoding() {
return encoding;
}
public String getMovieTitleIdentifier() {
return movieTitleIdentifier;
}
public String getMoviePosterIdentifier() {
return moviePosterIdentifier;
}
public String getRowTemplatePath() {
return rowTemplatePath;
}
public int getMoviesPerRow() {
return moviesPerRow;
}
public String getRowIdentigier() {
return rowIdentigier;
}
//#endregion
}
private void fillUpWithDefaults(Settings settings) {
if (settings.encoding == null || settings.encoding.trim().isEmpty()) {
settings.encoding = "utf-8";
}
if (settings.collectionTemplatePath == null || settings.collectionTemplatePath.isEmpty()) {
settings.collectionTemplatePath = getClass().getResource("/templates/collection.html").getPath();
}
if (settings.movieTemplatePath == null || settings.movieTemplatePath.isEmpty()) {
settings.movieTemplatePath = getClass().getResource("/templates/movie.html").getPath();
}
if (settings.rowTemplatePath == null || settings.rowTemplatePath.isEmpty()) {
settings.rowTemplatePath = getClass().getResource("/templates/row.html").getPath();
}
if (settings.collectionIdentifier == null || settings.collectionIdentifier.trim().isEmpty()) {
settings.collectionIdentifier = "{{collection}}";
}
if (settings.rowIdentigier == null || settings.rowIdentigier.isEmpty()) {
settings.rowIdentigier = "{{row}}";
}
if (settings.movieTitleIdentifier == null || settings.movieTitleIdentifier.isEmpty()) {
settings.movieTitleIdentifier = "{{title}}";
}
if (settings.moviePosterIdentifier == null || settings.moviePosterIdentifier.isEmpty()) {
settings.moviePosterIdentifier = "{{poster}}";
}
if (settings.movieFilenameIdentifier == null || settings.movieFilenameIdentifier.isEmpty()) {
settings.movieFilenameIdentifier = "{{filename}}";
}
if (settings.moviesPerRow <= 0) {
settings.moviesPerRow = 3;
}
}
private void start(String[] args) throws CollectionComposerException {
Settings settings = new Settings();
CmdLineParser parser = new CmdLineParser(settings);
try {
parser.parseArgument(args);
System.out.println("settings = " + settings);
} catch (CmdLineException e) {
System.err.println("e = " + e.toString());
parser.printUsage(System.out);
return;
}
fillUpWithDefaults(settings);
CollectionComposer composer = new CollectionComposer();
composer.compose(settings);
}
public static void main(String[] args) throws CollectionComposerException {
Runner runner = new Runner();
runner.start(args);
}
}

209
collection/src/main/java/ru/bvn13/imdbspider/collection/composer/CollectionComposer.java 100644
View File

@ -0,0 +1,209 @@
package ru.bvn13.imdbspider.collection.composer;
import ru.bvn13.imdbspider.collection.Runner;
import ru.bvn13.imdbspider.collection.exceptions.CollectionComposerException;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
/**
* Created by bvn13 on 27.01.2019.
*/
public class CollectionComposer {
private static final class ProcessFile extends SimpleFileVisitor<Path> {
private FileTreeElement treeElement;
private List<String> filters = new ArrayList<>();
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
boolean isApropriate = false;
for (String filter : filters) {
if (file.getFileName().toString().endsWith(filter) || file.getFileName().toString().matches(filter)) {
isApropriate = true;
break;
}
}
if (isApropriate) {
//System.out.println("Processing file:" + file);
FileTreeElement treeFile = new FileTreeElement();
treeFile.setParent(treeElement);
treeFile.setPath(file);
treeFile.setFullPath(file.toString());
treeFile.setType(FileTreeElement.TYPE.FILE);
treeElement.getNestedElements().add(treeFile);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
//System.out.println("Processing directory:" + dir);
if (treeElement == null) {
treeElement = new FileTreeElement();
} else if (!treeElement.getFullPath().equals(dir.toString())) {
FileTreeElement subTreeElement = new FileTreeElement();
subTreeElement.setParent(treeElement);
treeElement.getNestedElements().add(subTreeElement);
treeElement = subTreeElement;
}
treeElement.setPath(dir);
treeElement.setFullPath(dir.toString());
treeElement.setType(FileTreeElement.TYPE.DIRECTORY);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path path, IOException exc) throws IOException {
return FileVisitResult.SKIP_SUBTREE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
FileTreeElement current = treeElement;
treeElement = treeElement.getParent();
if (current.getNestedElements().size() == 0) {
treeElement.getNestedElements().remove(current);
current = null;
}
return FileVisitResult.CONTINUE;
}
public List<String> getFilters() {
return filters;
}
}
private FileTreeElement tree;
private String collectionTemplate;
private String movieTemplate;
private String rowTemplate;
private ExecutorService executorService;
public void compose(Runner.Settings settings) throws CollectionComposerException {
findAllVideoFiles(settings);
Charset charset = Charset.forName(settings.getEncoding());
try {
collectionTemplate = readFile(settings.getCollectionTemplatePath(), charset);
movieTemplate = readFile(settings.getMovieTemplatePath(), charset);
rowTemplate = readFile(settings.getRowTemplatePath(), charset);
} catch (IOException e) {
throw new CollectionComposerException("Error reading file", e);
}
AtomicInteger threadsCount = new AtomicInteger(0);
AtomicInteger completedCount = new AtomicInteger(0);
Object object = new Object();
executorService = Executors.newCachedThreadPool();
startMovieRetriever(tree, threadsCount, completedCount, fname -> {
synchronized (object) {
System.out.print(String.format("Completed: [%d/%d]\r", completedCount.get(), threadsCount.get()));
}
});
executorService.shutdown();
try {
executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
throw new CollectionComposerException("Could not retrieve movie data", e);
}
System.out.println("\n");
}
private void findAllVideoFiles(Runner.Settings settings) {
tree = new FileTreeElement();
tree.setFullPath(settings.getLibraryPath());
ProcessFile fileProcessor = new ProcessFile();
fileProcessor.treeElement = tree;
fileProcessor.getFilters().add(".+\\.3g2$");
fileProcessor.getFilters().add(".+\\.3gp$");
fileProcessor.getFilters().add(".+\\.3gp2$");
fileProcessor.getFilters().add(".+\\.3gpp$");
fileProcessor.getFilters().add(".+\\.3gpp2$");
fileProcessor.getFilters().add(".+\\.asf$");
fileProcessor.getFilters().add(".+\\.asx$");
fileProcessor.getFilters().add(".+\\.avi$");
fileProcessor.getFilters().add(".+\\.bin$");
fileProcessor.getFilters().add(".+\\.dat$");
fileProcessor.getFilters().add(".+\\.drv$");
fileProcessor.getFilters().add(".+\\.f4v$");
fileProcessor.getFilters().add(".+\\.flv$");
fileProcessor.getFilters().add(".+\\.gtp$");
fileProcessor.getFilters().add(".+\\.h264$");
fileProcessor.getFilters().add(".+\\.m4v$");
fileProcessor.getFilters().add(".+\\.mkv$");
fileProcessor.getFilters().add(".+\\.mod$");
fileProcessor.getFilters().add(".+\\.moov$");
fileProcessor.getFilters().add(".+\\.mov$");
fileProcessor.getFilters().add(".+\\.mp4$");
fileProcessor.getFilters().add(".+\\.mpeg$");
fileProcessor.getFilters().add(".+\\.mpg$");
fileProcessor.getFilters().add(".+\\.mts$");
fileProcessor.getFilters().add(".+\\.rm$");
fileProcessor.getFilters().add(".+\\.rmvb$");
fileProcessor.getFilters().add(".+\\.spl$");
fileProcessor.getFilters().add(".+\\.srt$");
fileProcessor.getFilters().add(".+\\.stl$");
fileProcessor.getFilters().add(".+\\.swf$");
fileProcessor.getFilters().add(".+\\.ts$");
fileProcessor.getFilters().add(".+\\.vcd$");
fileProcessor.getFilters().add(".+\\.vid$");
fileProcessor.getFilters().add(".+\\.vid$");
fileProcessor.getFilters().add(".+\\.vid$");
fileProcessor.getFilters().add(".+\\.vob$");
fileProcessor.getFilters().add(".+\\.webm$");
fileProcessor.getFilters().add(".+\\.wm$");
fileProcessor.getFilters().add(".+\\.wmv$");
fileProcessor.getFilters().add(".+\\.yuv$");
try {
Files.walkFileTree(Paths.get(settings.getLibraryPath()), fileProcessor);
} catch (IOException e) {
e.printStackTrace();
}
tree.print();
}
private static String readFile(String path, Charset encoding) throws IOException {
byte[] encoded = Files.readAllBytes(Paths.get(path));
return new String(encoded, encoding);
}
private void startMovieRetriever(FileTreeElement treeElement, AtomicInteger threadsCount, AtomicInteger completedCount, Consumer<String> callback) {
if (treeElement.isFile()) {
threadsCount.incrementAndGet();
executorService.submit(new MovieRetriever(treeElement, completedCount, callback));
} else if (treeElement.isDirectory()) {
for (FileTreeElement nested : treeElement.getNestedElements()) {
startMovieRetriever(nested, threadsCount, completedCount, callback);
}
}
}
}

96
collection/src/main/java/ru/bvn13/imdbspider/collection/composer/FileTreeElement.java 100644
View File

@ -0,0 +1,96 @@
package ru.bvn13.imdbspider.collection.composer;
import ru.bvn13.imdbspider.imdb.Movie;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
/**
* Created by bvn13 on 28.01.2019.
*/
public class FileTreeElement {
enum TYPE {
UNKNOWN, FILE, DIRECTORY
}
private TYPE type = TYPE.UNKNOWN;
private Path path;
private String fullPath;
private FileTreeElement parent;
private List<FileTreeElement> nestedElements = new ArrayList<>();
private int depth = 0;
private Movie movie;
//#region _GETTERS_SETTERS_
public Path getPath() {
return path;
}
public void setPath(Path path) {
this.path = path;
}
public String getFullPath() {
return fullPath;
}
public void setFullPath(String fullPath) {
this.fullPath = fullPath;
}
public FileTreeElement getParent() {
return parent;
}
public void setParent(FileTreeElement parent) {
this.parent = parent;
this.depth = parent.depth + 1;
}
public List<FileTreeElement> getNestedElements() {
return nestedElements;
}
public void setType(TYPE type) {
this.type = type;
}
public boolean isDirectory() {
return type.equals(TYPE.DIRECTORY);
}
public boolean isFile() {
return type.equals(TYPE.FILE);
}
public int getDepth() {
return depth;
}
public Movie getMovie() {
return movie;
}
public void setMovie(Movie movie) {
this.movie = movie;
}
//#endregion
public void print() {
StringBuilder value = new StringBuilder();
for (int i=0; i<depth; i++) {
value.append(" ");
}
value.append("-> ");
value.append(fullPath);
System.out.println(value);
for (FileTreeElement nested : nestedElements) {
nested.print();
}
}
}

54
collection/src/main/java/ru/bvn13/imdbspider/collection/composer/MovieRetriever.java 100644
View File

@ -0,0 +1,54 @@
package ru.bvn13.imdbspider.collection.composer;
import ru.bvn13.imdbspider.ImdbSpider;
import ru.bvn13.imdbspider.exceptions.ImdbSpiderException;
import ru.bvn13.imdbspider.imdb.MovieList;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
/**
* Created by bvn13 on 28.01.2019.
*/
public class MovieRetriever implements Runnable {
private FileTreeElement treeElement;
private AtomicInteger completedCount;
private Consumer<String> callback;
public MovieRetriever(FileTreeElement treeElement, AtomicInteger completedCount, Consumer<String> callback) {
this.treeElement = treeElement;
this.completedCount = completedCount;
this.callback = callback;
}
private String prepareFileName(String roughFileName) {
String filename = "";
return roughFileName;
}
@Override
public void run() {
String roughFilename = treeElement.getPath().getFileName().toString();
String filename = prepareFileName(roughFilename);
ImdbSpider spider = ImdbSpider.withApi_1_0();
try {
MovieList result = spider.searchMovieByTitle(filename);
} catch (ImdbSpiderException e) {
e.printStackTrace();
}
completedCount.incrementAndGet();
callback.accept(roughFilename);
}
}

25
collection/src/main/java/ru/bvn13/imdbspider/collection/exceptions/CollectionComposerException.java 100644
View File

@ -0,0 +1,25 @@
package ru.bvn13.imdbspider.collection.exceptions;
/**
* Created by bvn13 on 28.01.2019.
*/
public class CollectionComposerException extends Exception {
public CollectionComposerException() {
}
public CollectionComposerException(String message) {
super(message);
}
public CollectionComposerException(String message, Throwable cause) {
super(message, cause);
}
public CollectionComposerException(Throwable cause) {
super(cause);
}
public CollectionComposerException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}

28
collection/src/main/resources/templates/collection.html 100644
View File

@ -0,0 +1,28 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>bvn13's Collection</title>
<!--Import Google Icon Font-->
<link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
<!-- Compiled and minified CSS -->
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/css/materialize.min.css">
<!-- Compiled and minified JavaScript -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/js/materialize.min.js"></script>
<!--Let browser know website is optimized for mobile-->
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
</head>
<body>
<div class="container">
<div>{{collection}}</div>
</div>
</body>
</html>

7
collection/src/main/resources/templates/movie.html 100644
View File

@ -0,0 +1,7 @@
<div class="movie">
<h1>{{title}}</h1>
<h2>{{filename}}</h2>
<div><img src="{{poster}}"/></div>
</div>

3
collection/src/main/resources/templates/row.html 100644
View File

@ -0,0 +1,3 @@
<div class="row">
{{row}}
</div>

20
collection/src/test/java/ru/bvn13/imdbspider/collection/RunnerTest.java 100644
View File

@ -0,0 +1,20 @@
package ru.bvn13.imdbspider.collection;
import org.junit.Test;
import static org.junit.Assert.assertTrue;
/**
* Unit test for simple App.
*/
public class RunnerTest
{
/**
* Rigorous Test :-)
*/
@Test
public void shouldAnswerWithTrue()
{
assertTrue( true );
}
}

1
pom.xml
View File

@ -10,6 +10,7 @@
<modules>
<module>core</module>
<module>runner</module>
<module>collection</module>
</modules>
<name>IMDB-SPIDER :: PARENT</name>