mirror of https://github.com/bvn13/imdb-spider.git
implemented restriction by count for search movie by title request
parent
36dadace70
commit
49ed71c56a
|
@ -5,7 +5,7 @@
|
||||||
<parent>
|
<parent>
|
||||||
<artifactId>parent</artifactId>
|
<artifactId>parent</artifactId>
|
||||||
<groupId>ru.bvn13.imdbspider</groupId>
|
<groupId>ru.bvn13.imdbspider</groupId>
|
||||||
<version>1.0-SNAPSHOT</version>
|
<version>0.0.1</version>
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
|
@ -14,7 +14,10 @@ import ru.bvn13.imdbspider.spider.api.ApiFactory;
|
||||||
|
|
||||||
import java.net.URLEncoder;
|
import java.net.URLEncoder;
|
||||||
import java.nio.charset.Charset;
|
import java.nio.charset.Charset;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Date;
|
||||||
import java.util.EnumSet;
|
import java.util.EnumSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.ExecutionException;
|
import java.util.concurrent.ExecutionException;
|
||||||
|
@ -45,7 +48,7 @@ public class ImdbSpider {
|
||||||
}
|
}
|
||||||
|
|
||||||
public MovieList searchMovieByTitle(String title) throws ImdbSpiderException {
|
public MovieList searchMovieByTitle(String title) throws ImdbSpiderException {
|
||||||
return searchMovieByTitle(title, 10);
|
return searchMovieByTitle(title, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
public MovieList searchMovieByTitle(String title, int maxCount) throws ImdbSpiderException {
|
public MovieList searchMovieByTitle(String title, int maxCount) throws ImdbSpiderException {
|
||||||
|
@ -61,16 +64,23 @@ public class ImdbSpider {
|
||||||
try {
|
try {
|
||||||
Task t1 = apiFactory.taskByDataType(MovieListDataType.ELEMENTS);
|
Task t1 = apiFactory.taskByDataType(MovieListDataType.ELEMENTS);
|
||||||
t1.setUrl(url);
|
t1.setUrl(url);
|
||||||
|
if (maxCount > 0) {
|
||||||
|
t1.setRestrictionByCount(maxCount);
|
||||||
|
}
|
||||||
tasks.add(t1);
|
tasks.add(t1);
|
||||||
} catch (DataTypeNotSupportedException e) {
|
} catch (DataTypeNotSupportedException e) {
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
LocalDateTime dateStart = LocalDateTime.now();
|
||||||
try {
|
try {
|
||||||
manager.processTasks(tasks);
|
manager.processTasks(tasks);
|
||||||
} catch (ExecutionException | InterruptedException e) {
|
} catch (ExecutionException | InterruptedException e) {
|
||||||
throw new ImdbSpiderException("Error has been occurred!", e);
|
throw new ImdbSpiderException("Error has been occurred!", e);
|
||||||
}
|
}
|
||||||
|
LocalDateTime dateEnd = LocalDateTime.now();
|
||||||
|
Duration diff = Duration.between(dateStart, dateEnd);
|
||||||
|
System.out.println("TIME SPENT: "+(diff.toMillis())+" msec");
|
||||||
|
|
||||||
|
|
||||||
MovieListComposer movieListComposer = (MovieListComposer) imdbObjectComposerFactory.getComposer(MovieList.class);
|
MovieListComposer movieListComposer = (MovieListComposer) imdbObjectComposerFactory.getComposer(MovieList.class);
|
||||||
|
|
|
@ -93,7 +93,14 @@ public class ApiFactory_1_0 implements ApiFactory {
|
||||||
t.setCssSelector("#main > div > div.findSection > table > tbody > tr > td.result_text");
|
t.setCssSelector("#main > div > div.findSection > table > tbody > tr > td.result_text");
|
||||||
t.setResultType(List.class);
|
t.setResultType(List.class);
|
||||||
t.setPostprocess((task, s) -> {
|
t.setPostprocess((task, s) -> {
|
||||||
|
int count = 0;
|
||||||
for (Element element : task.getCssSelectorResult()) {
|
for (Element element : task.getCssSelectorResult()) {
|
||||||
|
count++;
|
||||||
|
if (task.getRestrictionByCount() != null) {
|
||||||
|
if (count > task.getRestrictionByCount()) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
Element link = element.select("a").first();
|
Element link = element.select("a").first();
|
||||||
if (!defaultMovieDataType.contains(MovieDataType.ID)) {
|
if (!defaultMovieDataType.contains(MovieDataType.ID)) {
|
||||||
defaultMovieDataType.add(MovieDataType.ID);
|
defaultMovieDataType.add(MovieDataType.ID);
|
||||||
|
|
|
@ -19,6 +19,8 @@ public class Task {
|
||||||
private String cssSelector;
|
private String cssSelector;
|
||||||
private Elements cssSelectorResult;
|
private Elements cssSelectorResult;
|
||||||
|
|
||||||
|
private Integer restrictionByCount;
|
||||||
|
|
||||||
private Class resultType;
|
private Class resultType;
|
||||||
private Object result;
|
private Object result;
|
||||||
|
|
||||||
|
@ -143,4 +145,13 @@ public class Task {
|
||||||
this.parentTask = parentTask;
|
this.parentTask = parentTask;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Integer getRestrictionByCount() {
|
||||||
|
return restrictionByCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task setRestrictionByCount(Integer restrictionByCount) {
|
||||||
|
this.restrictionByCount = restrictionByCount;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
2
pom.xml
2
pom.xml
|
@ -6,7 +6,7 @@
|
||||||
|
|
||||||
<groupId>ru.bvn13.imdbspider</groupId>
|
<groupId>ru.bvn13.imdbspider</groupId>
|
||||||
<artifactId>parent</artifactId>
|
<artifactId>parent</artifactId>
|
||||||
<version>1.0-SNAPSHOT</version>
|
<version>0.0.1</version>
|
||||||
<modules>
|
<modules>
|
||||||
<module>core</module>
|
<module>core</module>
|
||||||
<module>runner</module>
|
<module>runner</module>
|
||||||
|
|
|
@ -5,7 +5,7 @@
|
||||||
<parent>
|
<parent>
|
||||||
<artifactId>parent</artifactId>
|
<artifactId>parent</artifactId>
|
||||||
<groupId>ru.bvn13.imdbspider</groupId>
|
<groupId>ru.bvn13.imdbspider</groupId>
|
||||||
<version>1.0-SNAPSHOT</version>
|
<version>0.0.1</version>
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
@ -25,7 +25,7 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>ru.bvn13.imdbspider</groupId>
|
<groupId>ru.bvn13.imdbspider</groupId>
|
||||||
<artifactId>imdb-spider-core</artifactId>
|
<artifactId>imdb-spider-core</artifactId>
|
||||||
<version>1.0-SNAPSHOT</version>
|
<version>0.0.1</version>
|
||||||
<scope>compile</scope>
|
<scope>compile</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue