implemented retention policy for storage

develop
Vyacheslav Boyko 2019-03-12 17:16:20 +03:00
parent 6dcdb078cb
commit 46fea03041
14 changed files with 258 additions and 7 deletions

View File

@ -90,6 +90,7 @@
<orderEntry type="library" name="Maven: org.springframework:spring-tx:5.1.5.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-aspects:5.1.5.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.hsqldb:hsqldb:2.4.1" level="project" />
<orderEntry type="library" name="Maven: org.modelmapper:modelmapper:2.3.2" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-jetty:2.1.3.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-servlets:9.4.14.v20181114" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-continuation:9.4.14.v20181114" level="project" />

View File

@ -13,6 +13,15 @@
<module name="adastor" target="11" />
</bytecodeTargetLevel>
</component>
<component name="DataSourceManagerImpl" format="xml" multifile-model="true">
<data-source source="LOCAL" name="database" uuid="8f9d20f6-5642-4a5c-a61a-7743b445808b">
<driver-ref>hsqldb.local</driver-ref>
<synchronize>true</synchronize>
<configured-by-url>true</configured-by-url>
<jdbc-driver>org.hsqldb.jdbc.JDBCDriver</jdbc-driver>
<jdbc-url>jdbc:hsqldb:file:$PROJECT_DIR$/database/adastor</jdbc-url>
</data-source>
</component>
<component name="Encoding" addBOMForNewFiles="with NO BOM">
<file url="file://$PROJECT_DIR$" charset="UTF-8" />
<file url="PROJECT" charset="UTF-8" />
@ -167,6 +176,16 @@
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
<component name="dataSourceStorageLocal">
<data-source name="database" uuid="8f9d20f6-5642-4a5c-a61a-7743b445808b">
<database-info product="HSQL Database Engine" version="2.4.0" jdbc-version="4.2" driver-name="HSQL Database Engine Driver" driver-version="2.4.0" dbms="HSQLDB" exact-version="0">
<identifier-quote-string>&quot;</identifier-quote-string>
</database-info>
<case-sensitivity plain-identifiers="upper" quoted-identifiers="exact" />
<secret-storage>master_key</secret-storage>
<user-name>sa</user-name>
</data-source>
</component>
<component name="libraryTable">
<library name="Maven: antlr:antlr:2.7.7">
<CLASSES>
@ -883,6 +902,17 @@
<root url="jar://$MAVEN_REPOSITORY$/org/mockito/mockito-core/2.23.4/mockito-core-2.23.4-sources.jar!/" />
</SOURCES>
</library>
<library name="Maven: org.modelmapper:modelmapper:2.3.2">
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/org/modelmapper/modelmapper/2.3.2/modelmapper-2.3.2.jar!/" />
</CLASSES>
<JAVADOC>
<root url="jar://$MAVEN_REPOSITORY$/org/modelmapper/modelmapper/2.3.2/modelmapper-2.3.2-javadoc.jar!/" />
</JAVADOC>
<SOURCES>
<root url="jar://$MAVEN_REPOSITORY$/org/modelmapper/modelmapper/2.3.2/modelmapper-2.3.2-sources.jar!/" />
</SOURCES>
</library>
<library name="Maven: org.mortbay.jasper:apache-el:8.5.35.1">
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/org/mortbay/jasper/apache-el/8.5.35.1/apache-el-8.5.35.1.jar!/" />

View File

@ -42,6 +42,12 @@
<artifactId>hsqldb</artifactId>
</dependency>
<dependency>
<groupId>org.modelmapper</groupId>
<artifactId>modelmapper</artifactId>
<version>2.3.2</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jetty</artifactId>

View File

@ -4,6 +4,7 @@ import lombok.Getter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import javax.annotation.PostConstruct;
import java.io.File;
/**
@ -16,9 +17,38 @@ public class Config {
private String storagePath;
@Getter
// #{new Integer.parseInt('${api.orders.pingFrequency}')}
@Value("${adastor.storage.space.free}")
private Long freeSpace;
@Getter
@Value("${adastor.max-size}")
private Long maxSize;
@Getter
@Value("${adastor.min-days-storing}")
private Long minDaysStoring;
@Getter
@Value("${adastor.max-days-storing}")
private Long maxDaysStoring;
@PostConstruct
public void checkParams() {
if (storagePath == null || storagePath.isEmpty()) {
throw new IllegalArgumentException("Storage path is not specified!");
}
if (freeSpace == null || freeSpace.equals(0L)) {
throw new IllegalArgumentException("Free space is not specified!");
}
if (maxSize == null || maxSize.equals(0L)) {
throw new IllegalArgumentException("Max size is not specified!");
}
if (maxDaysStoring == null || maxDaysStoring.equals(0L)) {
throw new IllegalArgumentException("Max days storing is not specified!");
}
}
public String getStoragePath() {
String absStoragePath = storagePath.startsWith(".") ? new File("").getAbsolutePath() + storagePath.substring(1) : storagePath;
File path = new File(absStoragePath);

View File

@ -1,5 +1,7 @@
package ru.bvn13.adastor.config;
import org.modelmapper.ModelMapper;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
@ -8,5 +10,9 @@ import org.springframework.context.annotation.Configuration;
@Configuration
public class WebConfig {
@Bean
public ModelMapper modelMapper() {
return new ModelMapper();
}
}

View File

@ -4,10 +4,13 @@ import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.springframework.beans.factory.annotation.Autowired;
import ru.bvn13.adastor.config.Config;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Transient;
import java.time.LocalDateTime;
/**

View File

@ -0,0 +1,31 @@
package ru.bvn13.adastor.entities.dtos;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import javax.persistence.Transient;
import java.time.LocalDateTime;
/**
* @author boykovn at 12.03.2019
*/
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
public class StortionDto {
private String uuid;
private LocalDateTime storeDate;
private long size;
private String path;
@Transient
private long retention;
}

View File

@ -0,0 +1,36 @@
package ru.bvn13.adastor.web.controllers;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import ru.bvn13.adastor.entities.Stortion;
import ru.bvn13.adastor.entities.dtos.StortionDto;
import ru.bvn13.adastor.web.repositories.StortionRepository;
import ru.bvn13.adastor.web.services.StortionService;
import java.util.stream.Stream;
/**
* @author boykovn at 12.03.2019
*/
@Controller
public class TestController {
private StortionService stortionService;
@Autowired
public void setStortionRepository(StortionService stortionService) {
this.stortionService = stortionService;
}
@GetMapping("/t")
public @ResponseBody String test() {
Stream<StortionDto> stortions = stortionService.findAllSortedByRetention();
stortions.forEach(st -> System.out.println(String.format("%s - %s - %s - %s", st.getUuid(), st.getStoreDate(), st.getSize(), st.getRetention())));
return "done";
}
}

View File

@ -4,7 +4,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import ru.bvn13.adastor.entities.Stortion;
import ru.bvn13.adastor.entities.dtos.StortionDto;
import ru.bvn13.adastor.web.services.StortionService;
import javax.servlet.http.HttpServletRequest;
@ -24,7 +24,8 @@ public class UploadController {
}
@PostMapping(value="/a", produces = {"application/json"})
public @ResponseBody Stortion uploadData(HttpServletRequest request) throws IOException {
public @ResponseBody
StortionDto uploadData(HttpServletRequest request) throws IOException {
return stortionService.createStortion(request.getInputStream());
}

View File

@ -0,0 +1,14 @@
package ru.bvn13.adastor.web.repositories;
import ru.bvn13.adastor.entities.Stortion;
import java.util.stream.Stream;
/**
* @author boykovn at 12.03.2019
*/
public interface CustomStortionRepository {
Stream<Stortion> findAllSortedByRetention();
}

View File

@ -8,6 +8,6 @@ import ru.bvn13.adastor.entities.Stortion;
* @author boykovn at 11.03.2019
*/
@Repository
public interface StortionRepository extends JpaRepository<Stortion, String> {
public interface StortionRepository extends JpaRepository<Stortion, String>, CustomStortionRepository {
}

View File

@ -0,0 +1,57 @@
package ru.bvn13.adastor.web.repositories.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Repository;
import ru.bvn13.adastor.config.Config;
import ru.bvn13.adastor.entities.Stortion;
import ru.bvn13.adastor.web.repositories.CustomStortionRepository;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.persistence.Query;
import java.util.stream.Stream;
/**
* @author boykovn at 12.03.2019
*/
@Repository
public class StortionRepositoryImpl implements CustomStortionRepository {
private Config config;
@Autowired
public void setConfig(Config config) {
this.config = config;
}
@PersistenceContext
private EntityManager entityManager;
/**
* RETENTION = min_age + (-max_age + min_age) * pow((file_size / max_size - 1), 3)
*
* Implemets the query - selecting all stortions ordered by retention ascending
*
* SELECT * FROM STORTION
* ORDER BY 'retention' ASC
*
* @return queried collection
*/
@Override
public Stream<Stortion> findAllSortedByRetention() {
long min_age = config.getMinDaysStoring();
long max_age = config.getMaxDaysStoring();
long max_size = config.getMaxSize();
Query query = entityManager.createQuery("SELECT s FROM Stortion s ORDER BY "
+" (CAST(:min_age as double) + (- CAST(:max_age as double) + CAST(:min_age as double)) * POWER((CAST(s.size as double) / CAST(:max_size as double) - 1), 3)) ASC");
query.setParameter("min_age", (double)min_age);
query.setParameter("max_age", (double)max_age);
query.setParameter("max_size", max_size);
return (Stream<Stortion>) query.getResultStream();
}
}

View File

@ -1,15 +1,18 @@
package ru.bvn13.adastor.web.services;
import org.modelmapper.ModelMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import ru.bvn13.adastor.config.Config;
import ru.bvn13.adastor.entities.Stortion;
import ru.bvn13.adastor.entities.dtos.StortionDto;
import ru.bvn13.adastor.web.repositories.StortionRepository;
import java.io.*;
import java.time.LocalDateTime;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Stream;
/**
* @author boykovn at 11.03.2019
@ -19,6 +22,7 @@ public class StortionService {
private StortionRepository stortionRepository;
private Config config;
private ModelMapper modelMapper;
@Autowired
public void setStortionRepository(StortionRepository stortionRepository) {
@ -30,6 +34,11 @@ public class StortionService {
this.config = config;
}
@Autowired
public void setModelMapper(ModelMapper modelMapper) {
this.modelMapper = modelMapper;
}
public Optional<Stortion> findStortion(String uuid) {
return stortionRepository.findById(uuid);
}
@ -45,7 +54,7 @@ public class StortionService {
return targetStream;
}
public Stortion createStortion(InputStream is) throws IOException {
public StortionDto createStortion(InputStream is) throws IOException {
String uuid = UUID.randomUUID().toString();
String path = String.format("/%s", uuid);
String fullPath = String.format("%s/%s", config.getStoragePath(), uuid);
@ -62,7 +71,28 @@ public class StortionService {
stortion.setSize(bytesCount);
stortionRepository.save(stortion);
return stortion;
return convertToDto(stortion);
}
private StortionDto convertToDto(Stortion stortion) {
StortionDto stortionDto = modelMapper.map(stortion, StortionDto.class);
stortionDto.setRetention(computeRetention(stortion));
return stortionDto;
}
public Stream<StortionDto> findAllSortedByRetention() {
Stream<Stortion> stortions = stortionRepository.findAllSortedByRetention();
return stortions.map(this::convertToDto);
}
/**
* RETENTION = min_age + (-max_age + min_age) * pow((file_size / max_size - 1), 3)
* @return retention
*/
private long computeRetention(Stortion stortion) {
double retention = config.getMinDaysStoring()
+ (-config.getMaxDaysStoring() + config.getMinDaysStoring()) * Math.pow((double) stortion.getSize() / config.getMaxSize() - 1, 3);
return Math.round(retention);
}
}

View File

@ -1,7 +1,13 @@
#relative or absolute
adastor.storage.path=./storage
#in Megabytes
adastor.storage.space.free=100Mb
#in bytes
adastor.storage.space.free=200000000
#max stortion size in bytes
adastor.max-size=100000000
#min days storing
adastor.min-days-storing=30
#max days storing
adastor.max-days-storing=356
spring.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver
spring.datasource.url=jdbc:hsqldb:file:./database/adastor