mirror of https://github.com/bvn13/ADaStor.git
implemented retention policy for storage
parent
6dcdb078cb
commit
46fea03041
|
@ -90,6 +90,7 @@
|
|||
<orderEntry type="library" name="Maven: org.springframework:spring-tx:5.1.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework:spring-aspects:5.1.5.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.hsqldb:hsqldb:2.4.1" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.modelmapper:modelmapper:2.3.2" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-jetty:2.1.3.RELEASE" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-servlets:9.4.14.v20181114" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-continuation:9.4.14.v20181114" level="project" />
|
||||
|
|
30
adastor.ipr
30
adastor.ipr
|
@ -13,6 +13,15 @@
|
|||
<module name="adastor" target="11" />
|
||||
</bytecodeTargetLevel>
|
||||
</component>
|
||||
<component name="DataSourceManagerImpl" format="xml" multifile-model="true">
|
||||
<data-source source="LOCAL" name="database" uuid="8f9d20f6-5642-4a5c-a61a-7743b445808b">
|
||||
<driver-ref>hsqldb.local</driver-ref>
|
||||
<synchronize>true</synchronize>
|
||||
<configured-by-url>true</configured-by-url>
|
||||
<jdbc-driver>org.hsqldb.jdbc.JDBCDriver</jdbc-driver>
|
||||
<jdbc-url>jdbc:hsqldb:file:$PROJECT_DIR$/database/adastor</jdbc-url>
|
||||
</data-source>
|
||||
</component>
|
||||
<component name="Encoding" addBOMForNewFiles="with NO BOM">
|
||||
<file url="file://$PROJECT_DIR$" charset="UTF-8" />
|
||||
<file url="PROJECT" charset="UTF-8" />
|
||||
|
@ -167,6 +176,16 @@
|
|||
<component name="VcsDirectoryMappings">
|
||||
<mapping directory="$PROJECT_DIR$" vcs="Git" />
|
||||
</component>
|
||||
<component name="dataSourceStorageLocal">
|
||||
<data-source name="database" uuid="8f9d20f6-5642-4a5c-a61a-7743b445808b">
|
||||
<database-info product="HSQL Database Engine" version="2.4.0" jdbc-version="4.2" driver-name="HSQL Database Engine Driver" driver-version="2.4.0" dbms="HSQLDB" exact-version="0">
|
||||
<identifier-quote-string>"</identifier-quote-string>
|
||||
</database-info>
|
||||
<case-sensitivity plain-identifiers="upper" quoted-identifiers="exact" />
|
||||
<secret-storage>master_key</secret-storage>
|
||||
<user-name>sa</user-name>
|
||||
</data-source>
|
||||
</component>
|
||||
<component name="libraryTable">
|
||||
<library name="Maven: antlr:antlr:2.7.7">
|
||||
<CLASSES>
|
||||
|
@ -883,6 +902,17 @@
|
|||
<root url="jar://$MAVEN_REPOSITORY$/org/mockito/mockito-core/2.23.4/mockito-core-2.23.4-sources.jar!/" />
|
||||
</SOURCES>
|
||||
</library>
|
||||
<library name="Maven: org.modelmapper:modelmapper:2.3.2">
|
||||
<CLASSES>
|
||||
<root url="jar://$MAVEN_REPOSITORY$/org/modelmapper/modelmapper/2.3.2/modelmapper-2.3.2.jar!/" />
|
||||
</CLASSES>
|
||||
<JAVADOC>
|
||||
<root url="jar://$MAVEN_REPOSITORY$/org/modelmapper/modelmapper/2.3.2/modelmapper-2.3.2-javadoc.jar!/" />
|
||||
</JAVADOC>
|
||||
<SOURCES>
|
||||
<root url="jar://$MAVEN_REPOSITORY$/org/modelmapper/modelmapper/2.3.2/modelmapper-2.3.2-sources.jar!/" />
|
||||
</SOURCES>
|
||||
</library>
|
||||
<library name="Maven: org.mortbay.jasper:apache-el:8.5.35.1">
|
||||
<CLASSES>
|
||||
<root url="jar://$MAVEN_REPOSITORY$/org/mortbay/jasper/apache-el/8.5.35.1/apache-el-8.5.35.1.jar!/" />
|
||||
|
|
6
pom.xml
6
pom.xml
|
@ -42,6 +42,12 @@
|
|||
<artifactId>hsqldb</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.modelmapper</groupId>
|
||||
<artifactId>modelmapper</artifactId>
|
||||
<version>2.3.2</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-jetty</artifactId>
|
||||
|
|
|
@ -4,6 +4,7 @@ import lombok.Getter;
|
|||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.io.File;
|
||||
|
||||
/**
|
||||
|
@ -16,9 +17,38 @@ public class Config {
|
|||
private String storagePath;
|
||||
|
||||
@Getter
|
||||
// #{new Integer.parseInt('${api.orders.pingFrequency}')}
|
||||
@Value("${adastor.storage.space.free}")
|
||||
private Long freeSpace;
|
||||
|
||||
@Getter
|
||||
@Value("${adastor.max-size}")
|
||||
private Long maxSize;
|
||||
|
||||
@Getter
|
||||
@Value("${adastor.min-days-storing}")
|
||||
private Long minDaysStoring;
|
||||
|
||||
@Getter
|
||||
@Value("${adastor.max-days-storing}")
|
||||
private Long maxDaysStoring;
|
||||
|
||||
@PostConstruct
|
||||
public void checkParams() {
|
||||
if (storagePath == null || storagePath.isEmpty()) {
|
||||
throw new IllegalArgumentException("Storage path is not specified!");
|
||||
}
|
||||
if (freeSpace == null || freeSpace.equals(0L)) {
|
||||
throw new IllegalArgumentException("Free space is not specified!");
|
||||
}
|
||||
if (maxSize == null || maxSize.equals(0L)) {
|
||||
throw new IllegalArgumentException("Max size is not specified!");
|
||||
}
|
||||
if (maxDaysStoring == null || maxDaysStoring.equals(0L)) {
|
||||
throw new IllegalArgumentException("Max days storing is not specified!");
|
||||
}
|
||||
}
|
||||
|
||||
public String getStoragePath() {
|
||||
String absStoragePath = storagePath.startsWith(".") ? new File("").getAbsolutePath() + storagePath.substring(1) : storagePath;
|
||||
File path = new File(absStoragePath);
|
||||
|
|
|
@ -1,5 +1,7 @@
|
|||
package ru.bvn13.adastor.config;
|
||||
|
||||
import org.modelmapper.ModelMapper;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
|
@ -8,5 +10,9 @@ import org.springframework.context.annotation.Configuration;
|
|||
@Configuration
|
||||
public class WebConfig {
|
||||
|
||||
@Bean
|
||||
public ModelMapper modelMapper() {
|
||||
return new ModelMapper();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -4,10 +4,13 @@ import lombok.AllArgsConstructor;
|
|||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import ru.bvn13.adastor.config.Config;
|
||||
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Transient;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
|
|
|
@ -0,0 +1,31 @@
|
|||
package ru.bvn13.adastor.entities.dtos;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
import javax.persistence.Transient;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* @author boykovn at 12.03.2019
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class StortionDto {
|
||||
|
||||
private String uuid;
|
||||
|
||||
private LocalDateTime storeDate;
|
||||
|
||||
private long size;
|
||||
|
||||
private String path;
|
||||
|
||||
@Transient
|
||||
private long retention;
|
||||
|
||||
}
|
|
@ -0,0 +1,36 @@
|
|||
package ru.bvn13.adastor.web.controllers;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Controller;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.ResponseBody;
|
||||
import ru.bvn13.adastor.entities.Stortion;
|
||||
import ru.bvn13.adastor.entities.dtos.StortionDto;
|
||||
import ru.bvn13.adastor.web.repositories.StortionRepository;
|
||||
import ru.bvn13.adastor.web.services.StortionService;
|
||||
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
* @author boykovn at 12.03.2019
|
||||
*/
|
||||
@Controller
|
||||
public class TestController {
|
||||
|
||||
private StortionService stortionService;
|
||||
|
||||
@Autowired
|
||||
public void setStortionRepository(StortionService stortionService) {
|
||||
this.stortionService = stortionService;
|
||||
}
|
||||
|
||||
@GetMapping("/t")
|
||||
public @ResponseBody String test() {
|
||||
|
||||
Stream<StortionDto> stortions = stortionService.findAllSortedByRetention();
|
||||
stortions.forEach(st -> System.out.println(String.format("%s - %s - %s - %s", st.getUuid(), st.getStoreDate(), st.getSize(), st.getRetention())));
|
||||
|
||||
return "done";
|
||||
}
|
||||
|
||||
}
|
|
@ -4,7 +4,7 @@ import org.springframework.beans.factory.annotation.Autowired;
|
|||
import org.springframework.stereotype.Controller;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.ResponseBody;
|
||||
import ru.bvn13.adastor.entities.Stortion;
|
||||
import ru.bvn13.adastor.entities.dtos.StortionDto;
|
||||
import ru.bvn13.adastor.web.services.StortionService;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
@ -24,7 +24,8 @@ public class UploadController {
|
|||
}
|
||||
|
||||
@PostMapping(value="/a", produces = {"application/json"})
|
||||
public @ResponseBody Stortion uploadData(HttpServletRequest request) throws IOException {
|
||||
public @ResponseBody
|
||||
StortionDto uploadData(HttpServletRequest request) throws IOException {
|
||||
return stortionService.createStortion(request.getInputStream());
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,14 @@
|
|||
package ru.bvn13.adastor.web.repositories;
|
||||
|
||||
import ru.bvn13.adastor.entities.Stortion;
|
||||
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
* @author boykovn at 12.03.2019
|
||||
*/
|
||||
public interface CustomStortionRepository {
|
||||
|
||||
Stream<Stortion> findAllSortedByRetention();
|
||||
|
||||
}
|
|
@ -8,6 +8,6 @@ import ru.bvn13.adastor.entities.Stortion;
|
|||
* @author boykovn at 11.03.2019
|
||||
*/
|
||||
@Repository
|
||||
public interface StortionRepository extends JpaRepository<Stortion, String> {
|
||||
public interface StortionRepository extends JpaRepository<Stortion, String>, CustomStortionRepository {
|
||||
|
||||
}
|
||||
|
|
|
@ -0,0 +1,57 @@
|
|||
package ru.bvn13.adastor.web.repositories.impl;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Repository;
|
||||
import ru.bvn13.adastor.config.Config;
|
||||
import ru.bvn13.adastor.entities.Stortion;
|
||||
import ru.bvn13.adastor.web.repositories.CustomStortionRepository;
|
||||
|
||||
import javax.persistence.EntityManager;
|
||||
import javax.persistence.PersistenceContext;
|
||||
import javax.persistence.Query;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
* @author boykovn at 12.03.2019
|
||||
*/
|
||||
@Repository
|
||||
public class StortionRepositoryImpl implements CustomStortionRepository {
|
||||
|
||||
private Config config;
|
||||
|
||||
@Autowired
|
||||
public void setConfig(Config config) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
|
||||
@PersistenceContext
|
||||
private EntityManager entityManager;
|
||||
|
||||
/**
|
||||
* RETENTION = min_age + (-max_age + min_age) * pow((file_size / max_size - 1), 3)
|
||||
*
|
||||
* Implemets the query - selecting all stortions ordered by retention ascending
|
||||
*
|
||||
* SELECT * FROM STORTION
|
||||
* ORDER BY 'retention' ASC
|
||||
*
|
||||
* @return queried collection
|
||||
*/
|
||||
@Override
|
||||
public Stream<Stortion> findAllSortedByRetention() {
|
||||
|
||||
long min_age = config.getMinDaysStoring();
|
||||
long max_age = config.getMaxDaysStoring();
|
||||
long max_size = config.getMaxSize();
|
||||
|
||||
Query query = entityManager.createQuery("SELECT s FROM Stortion s ORDER BY "
|
||||
+" (CAST(:min_age as double) + (- CAST(:max_age as double) + CAST(:min_age as double)) * POWER((CAST(s.size as double) / CAST(:max_size as double) - 1), 3)) ASC");
|
||||
query.setParameter("min_age", (double)min_age);
|
||||
query.setParameter("max_age", (double)max_age);
|
||||
query.setParameter("max_size", max_size);
|
||||
|
||||
return (Stream<Stortion>) query.getResultStream();
|
||||
|
||||
}
|
||||
}
|
|
@ -1,15 +1,18 @@
|
|||
package ru.bvn13.adastor.web.services;
|
||||
|
||||
import org.modelmapper.ModelMapper;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import ru.bvn13.adastor.config.Config;
|
||||
import ru.bvn13.adastor.entities.Stortion;
|
||||
import ru.bvn13.adastor.entities.dtos.StortionDto;
|
||||
import ru.bvn13.adastor.web.repositories.StortionRepository;
|
||||
|
||||
import java.io.*;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
* @author boykovn at 11.03.2019
|
||||
|
@ -19,6 +22,7 @@ public class StortionService {
|
|||
|
||||
private StortionRepository stortionRepository;
|
||||
private Config config;
|
||||
private ModelMapper modelMapper;
|
||||
|
||||
@Autowired
|
||||
public void setStortionRepository(StortionRepository stortionRepository) {
|
||||
|
@ -30,6 +34,11 @@ public class StortionService {
|
|||
this.config = config;
|
||||
}
|
||||
|
||||
@Autowired
|
||||
public void setModelMapper(ModelMapper modelMapper) {
|
||||
this.modelMapper = modelMapper;
|
||||
}
|
||||
|
||||
public Optional<Stortion> findStortion(String uuid) {
|
||||
return stortionRepository.findById(uuid);
|
||||
}
|
||||
|
@ -45,7 +54,7 @@ public class StortionService {
|
|||
return targetStream;
|
||||
}
|
||||
|
||||
public Stortion createStortion(InputStream is) throws IOException {
|
||||
public StortionDto createStortion(InputStream is) throws IOException {
|
||||
String uuid = UUID.randomUUID().toString();
|
||||
String path = String.format("/%s", uuid);
|
||||
String fullPath = String.format("%s/%s", config.getStoragePath(), uuid);
|
||||
|
@ -62,7 +71,28 @@ public class StortionService {
|
|||
stortion.setSize(bytesCount);
|
||||
|
||||
stortionRepository.save(stortion);
|
||||
return stortion;
|
||||
return convertToDto(stortion);
|
||||
}
|
||||
|
||||
private StortionDto convertToDto(Stortion stortion) {
|
||||
StortionDto stortionDto = modelMapper.map(stortion, StortionDto.class);
|
||||
stortionDto.setRetention(computeRetention(stortion));
|
||||
return stortionDto;
|
||||
}
|
||||
|
||||
public Stream<StortionDto> findAllSortedByRetention() {
|
||||
Stream<Stortion> stortions = stortionRepository.findAllSortedByRetention();
|
||||
return stortions.map(this::convertToDto);
|
||||
}
|
||||
|
||||
/**
|
||||
* RETENTION = min_age + (-max_age + min_age) * pow((file_size / max_size - 1), 3)
|
||||
* @return retention
|
||||
*/
|
||||
private long computeRetention(Stortion stortion) {
|
||||
double retention = config.getMinDaysStoring()
|
||||
+ (-config.getMaxDaysStoring() + config.getMinDaysStoring()) * Math.pow((double) stortion.getSize() / config.getMaxSize() - 1, 3);
|
||||
return Math.round(retention);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -1,7 +1,13 @@
|
|||
#relative or absolute
|
||||
adastor.storage.path=./storage
|
||||
#in Megabytes
|
||||
adastor.storage.space.free=100Mb
|
||||
#in bytes
|
||||
adastor.storage.space.free=200000000
|
||||
#max stortion size in bytes
|
||||
adastor.max-size=100000000
|
||||
#min days storing
|
||||
adastor.min-days-storing=30
|
||||
#max days storing
|
||||
adastor.max-days-storing=356
|
||||
|
||||
spring.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver
|
||||
spring.datasource.url=jdbc:hsqldb:file:./database/adastor
|
||||
|
|
Loading…
Reference in New Issue