bvn13
/
imdb-spider
mirror of https://github.com/bvn13/imdb-spider.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

implemented restriction by count for search movie by title request

master
Vyacheslav Boyko 2019-01-11 10:08:50 +03:00
parent 36dadace70
commit 49ed71c56a
6 changed files with 33 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/pom.xml
View File

@ -5,7 +5,7 @@
<parent>
<artifactId>parent</artifactId>
<groupId>ru.bvn13.imdbspider</groupId>
<version>1.0-SNAPSHOT</version>
<version>0.0.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>

12
core/src/main/java/ru/bvn13/imdbspider/ImdbSpider.java
View File

@ -14,7 +14,10 @@ import ru.bvn13.imdbspider.spider.api.ApiFactory;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Date;
import java.util.EnumSet;
import java.util.List;
import java.util.concurrent.ExecutionException;
@ -45,7 +48,7 @@ public class ImdbSpider {
}
public MovieList searchMovieByTitle(String title) throws ImdbSpiderException {
return searchMovieByTitle(title, 10);
return searchMovieByTitle(title, 0);
}
public MovieList searchMovieByTitle(String title, int maxCount) throws ImdbSpiderException {
@ -61,16 +64,23 @@ public class ImdbSpider {
try {
Task t1 = apiFactory.taskByDataType(MovieListDataType.ELEMENTS);
t1.setUrl(url);
if (maxCount > 0) {
t1.setRestrictionByCount(maxCount);
}
tasks.add(t1);
} catch (DataTypeNotSupportedException e) {
throw e;
}
LocalDateTime dateStart = LocalDateTime.now();
try {
manager.processTasks(tasks);
} catch (ExecutionException | InterruptedException e) {
throw new ImdbSpiderException("Error has been occurred!", e);
}
LocalDateTime dateEnd = LocalDateTime.now();
Duration diff = Duration.between(dateStart, dateEnd);
System.out.println("TIME SPENT: "+(diff.toMillis())+" msec");
MovieListComposer movieListComposer = (MovieListComposer) imdbObjectComposerFactory.getComposer(MovieList.class);

7
core/src/main/java/ru/bvn13/imdbspider/spider/api/v1_0/ApiFactory_1_0.java
View File

@ -93,7 +93,14 @@ public class ApiFactory_1_0 implements ApiFactory {
t.setCssSelector("#main > div > div.findSection > table > tbody > tr > td.result_text");
t.setResultType(List.class);
t.setPostprocess((task, s) -> {
int count = 0;
for (Element element : task.getCssSelectorResult()) {
count++;
if (task.getRestrictionByCount() != null) {
if (count > task.getRestrictionByCount()) {
break;
}
}
Element link = element.select("a").first();
if (!defaultMovieDataType.contains(MovieDataType.ID)) {
defaultMovieDataType.add(MovieDataType.ID);

11
core/src/main/java/ru/bvn13/imdbspider/spider/tasker/Task.java
View File

@ -19,6 +19,8 @@ public class Task {
private String cssSelector;
private Elements cssSelectorResult;
private Integer restrictionByCount;
private Class resultType;
private Object result;
@ -143,4 +145,13 @@ public class Task {
this.parentTask = parentTask;
return this;
}
public Integer getRestrictionByCount() {
return restrictionByCount;
}
public Task setRestrictionByCount(Integer restrictionByCount) {
this.restrictionByCount = restrictionByCount;
return this;
}
}

2
pom.xml
View File

@ -6,7 +6,7 @@
<groupId>ru.bvn13.imdbspider</groupId>
<artifactId>parent</artifactId>
<version>1.0-SNAPSHOT</version>
<version>0.0.1</version>
<modules>
<module>core</module>
<module>runner</module>

4
runner/pom.xml
View File

@ -5,7 +5,7 @@
<parent>
<artifactId>parent</artifactId>
<groupId>ru.bvn13.imdbspider</groupId>
<version>1.0-SNAPSHOT</version>
<version>0.0.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@ -25,7 +25,7 @@
<dependency>
<groupId>ru.bvn13.imdbspider</groupId>
<artifactId>imdb-spider-core</artifactId>
<version>1.0-SNAPSHOT</version>
<version>0.0.1</version>
<scope>compile</scope>
</dependency>