implemented retention policy for storage

adastor.iml

@@ -90,6 +90,7 @@
     <orderEntry type="library" name="Maven: org.springframework:spring-tx:5.1.5.RELEASE" level="project" />
     <orderEntry type="library" name="Maven: org.springframework:spring-aspects:5.1.5.RELEASE" level="project" />
     <orderEntry type="library" name="Maven: org.hsqldb:hsqldb:2.4.1" level="project" />
+    <orderEntry type="library" name="Maven: org.modelmapper:modelmapper:2.3.2" level="project" />
     <orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-jetty:2.1.3.RELEASE" level="project" />
     <orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-servlets:9.4.14.v20181114" level="project" />
     <orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-continuation:9.4.14.v20181114" level="project" />

adastor.ipr

@@ -13,6 +13,15 @@
       <module name="adastor" target="11" />
     </bytecodeTargetLevel>
   </component>
+  <component name="DataSourceManagerImpl" format="xml" multifile-model="true">
+    <data-source source="LOCAL" name="database" uuid="8f9d20f6-5642-4a5c-a61a-7743b445808b">
+      <driver-ref>hsqldb.local</driver-ref>
+      <synchronize>true</synchronize>
+      <configured-by-url>true</configured-by-url>
+      <jdbc-driver>org.hsqldb.jdbc.JDBCDriver</jdbc-driver>
+      <jdbc-url>jdbc:hsqldb:file:$PROJECT_DIR$/database/adastor</jdbc-url>
+    </data-source>
+  </component>
   <component name="Encoding" addBOMForNewFiles="with NO BOM">
     <file url="file://$PROJECT_DIR$" charset="UTF-8" />
     <file url="PROJECT" charset="UTF-8" />
@@ -167,6 +176,16 @@
   <component name="VcsDirectoryMappings">
     <mapping directory="$PROJECT_DIR$" vcs="Git" />
   </component>
+  <component name="dataSourceStorageLocal">
+    <data-source name="database" uuid="8f9d20f6-5642-4a5c-a61a-7743b445808b">
+      <database-info product="HSQL Database Engine" version="2.4.0" jdbc-version="4.2" driver-name="HSQL Database Engine Driver" driver-version="2.4.0" dbms="HSQLDB" exact-version="0">
+        <identifier-quote-string>&quot;</identifier-quote-string>
+      </database-info>
+      <case-sensitivity plain-identifiers="upper" quoted-identifiers="exact" />
+      <secret-storage>master_key</secret-storage>
+      <user-name>sa</user-name>
+    </data-source>
+  </component>
   <component name="libraryTable">
     <library name="Maven: antlr:antlr:2.7.7">
       <CLASSES>
@@ -883,6 +902,17 @@
         <root url="jar://$MAVEN_REPOSITORY$/org/mockito/mockito-core/2.23.4/mockito-core-2.23.4-sources.jar!/" />
       </SOURCES>
     </library>
+    <library name="Maven: org.modelmapper:modelmapper:2.3.2">
+      <CLASSES>
+        <root url="jar://$MAVEN_REPOSITORY$/org/modelmapper/modelmapper/2.3.2/modelmapper-2.3.2.jar!/" />
+      </CLASSES>
+      <JAVADOC>
+        <root url="jar://$MAVEN_REPOSITORY$/org/modelmapper/modelmapper/2.3.2/modelmapper-2.3.2-javadoc.jar!/" />
+      </JAVADOC>
+      <SOURCES>
+        <root url="jar://$MAVEN_REPOSITORY$/org/modelmapper/modelmapper/2.3.2/modelmapper-2.3.2-sources.jar!/" />
+      </SOURCES>
+    </library>
     <library name="Maven: org.mortbay.jasper:apache-el:8.5.35.1">
       <CLASSES>
         <root url="jar://$MAVEN_REPOSITORY$/org/mortbay/jasper/apache-el/8.5.35.1/apache-el-8.5.35.1.jar!/" />

pom.xml

@@ -42,6 +42,12 @@
             <artifactId>hsqldb</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.modelmapper</groupId>
+            <artifactId>modelmapper</artifactId>
+            <version>2.3.2</version>
+        </dependency>
+
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-jetty</artifactId>

src/main/java/ru/bvn13/adastor/config/Config.java

@@ -4,6 +4,7 @@ import lombok.Getter;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Configuration;
 
+import javax.annotation.PostConstruct;
 import java.io.File;
 
 /**
@@ -16,9 +17,38 @@ public class Config {
     private String storagePath;
 
     @Getter
+    // #{new Integer.parseInt('${api.orders.pingFrequency}')}
     @Value("${adastor.storage.space.free}")
     private Long freeSpace;
 
+    @Getter
+    @Value("${adastor.max-size}")
+    private Long maxSize;
+
+    @Getter
+    @Value("${adastor.min-days-storing}")
+    private Long minDaysStoring;
+
+    @Getter
+    @Value("${adastor.max-days-storing}")
+    private Long maxDaysStoring;
+
+    @PostConstruct
+    public void checkParams() {
+        if (storagePath == null || storagePath.isEmpty()) {
+            throw new IllegalArgumentException("Storage path is not specified!");
+        }
+        if (freeSpace == null || freeSpace.equals(0L)) {
+            throw new IllegalArgumentException("Free space is not specified!");
+        }
+        if (maxSize == null || maxSize.equals(0L)) {
+            throw new IllegalArgumentException("Max size is not specified!");
+        }
+        if (maxDaysStoring == null || maxDaysStoring.equals(0L)) {
+            throw new IllegalArgumentException("Max days storing is not specified!");
+        }
+    }
+
     public String getStoragePath() {
         String absStoragePath = storagePath.startsWith(".") ? new File("").getAbsolutePath() + storagePath.substring(1) : storagePath;
         File path = new File(absStoragePath);

src/main/java/ru/bvn13/adastor/config/WebConfig.java

@@ -1,5 +1,7 @@
 package ru.bvn13.adastor.config;
 
+import org.modelmapper.ModelMapper;
+import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
 /**
@@ -8,5 +10,9 @@ import org.springframework.context.annotation.Configuration;
 @Configuration
 public class WebConfig {
 
+    @Bean
+    public ModelMapper modelMapper() {
+        return new ModelMapper();
+    }
 
 }

src/main/java/ru/bvn13/adastor/entities/Stortion.java

@@ -4,10 +4,13 @@ import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
+import org.springframework.beans.factory.annotation.Autowired;
+import ru.bvn13.adastor.config.Config;
 
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.Id;
+import javax.persistence.Transient;
 import java.time.LocalDateTime;
 
 /**

src/main/java/ru/bvn13/adastor/entities/dtos/StortionDto.java

@@ -0,0 +1,31 @@
+package ru.bvn13.adastor.entities.dtos;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import javax.persistence.Transient;
+import java.time.LocalDateTime;
+
+/**
+ * @author boykovn at 12.03.2019
+ */
+@Getter
+@Setter
+@AllArgsConstructor
+@NoArgsConstructor
+public class StortionDto {
+
+    private String uuid;
+
+    private LocalDateTime storeDate;
+
+    private long size;
+
+    private String path;
+
+    @Transient
+    private long retention;
+
+}

src/main/java/ru/bvn13/adastor/web/controllers/TestController.java

@@ -0,0 +1,36 @@
+package ru.bvn13.adastor.web.controllers;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.ResponseBody;
+import ru.bvn13.adastor.entities.Stortion;
+import ru.bvn13.adastor.entities.dtos.StortionDto;
+import ru.bvn13.adastor.web.repositories.StortionRepository;
+import ru.bvn13.adastor.web.services.StortionService;
+
+import java.util.stream.Stream;
+
+/**
+ * @author boykovn at 12.03.2019
+ */
+@Controller
+public class TestController {
+
+    private StortionService stortionService;
+
+    @Autowired
+    public void setStortionRepository(StortionService stortionService) {
+        this.stortionService = stortionService;
+    }
+
+    @GetMapping("/t")
+    public @ResponseBody String test() {
+
+        Stream<StortionDto> stortions = stortionService.findAllSortedByRetention();
+        stortions.forEach(st -> System.out.println(String.format("%s - %s - %s - %s", st.getUuid(), st.getStoreDate(), st.getSize(), st.getRetention())));
+
+        return "done";
+    }
+
+}

src/main/java/ru/bvn13/adastor/web/controllers/UploadController.java

@@ -4,7 +4,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.ResponseBody;
-import ru.bvn13.adastor.entities.Stortion;
+import ru.bvn13.adastor.entities.dtos.StortionDto;
 import ru.bvn13.adastor.web.services.StortionService;
 
 import javax.servlet.http.HttpServletRequest;
@@ -24,7 +24,8 @@ public class UploadController {
     }
 
     @PostMapping(value="/a", produces = {"application/json"})
-    public @ResponseBody Stortion uploadData(HttpServletRequest request) throws IOException {
+    public @ResponseBody
+    StortionDto uploadData(HttpServletRequest request) throws IOException {
         return stortionService.createStortion(request.getInputStream());
     }
 

src/main/java/ru/bvn13/adastor/web/repositories/CustomStortionRepository.java

@@ -0,0 +1,14 @@
+package ru.bvn13.adastor.web.repositories;
+
+import ru.bvn13.adastor.entities.Stortion;
+
+import java.util.stream.Stream;
+
+/**
+ * @author boykovn at 12.03.2019
+ */
+public interface CustomStortionRepository {
+
+    Stream<Stortion> findAllSortedByRetention();
+
+}

src/main/java/ru/bvn13/adastor/web/repositories/StortionRepository.java

@@ -8,6 +8,6 @@ import ru.bvn13.adastor.entities.Stortion;
  * @author boykovn at 11.03.2019
  */
 @Repository
-public interface StortionRepository extends JpaRepository<Stortion, String> {
+public interface StortionRepository extends JpaRepository<Stortion, String>, CustomStortionRepository {
 
 }

src/main/java/ru/bvn13/adastor/web/repositories/impl/StortionRepositoryImpl.java

@@ -0,0 +1,57 @@
+package ru.bvn13.adastor.web.repositories.impl;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Repository;
+import ru.bvn13.adastor.config.Config;
+import ru.bvn13.adastor.entities.Stortion;
+import ru.bvn13.adastor.web.repositories.CustomStortionRepository;
+
+import javax.persistence.EntityManager;
+import javax.persistence.PersistenceContext;
+import javax.persistence.Query;
+import java.util.stream.Stream;
+
+/**
+ * @author boykovn at 12.03.2019
+ */
+@Repository
+public class StortionRepositoryImpl implements CustomStortionRepository {
+
+    private Config config;
+
+    @Autowired
+    public void setConfig(Config config) {
+        this.config = config;
+    }
+
+
+    @PersistenceContext
+    private EntityManager entityManager;
+
+    /**
+     * RETENTION = min_age + (-max_age + min_age) * pow((file_size / max_size - 1), 3)
+     *
+     * Implemets the query - selecting all stortions ordered by retention ascending
+     *
+     * SELECT * FROM STORTION
+     * ORDER BY 'retention' ASC
+     *
+     * @return queried collection
+     */
+    @Override
+    public Stream<Stortion> findAllSortedByRetention() {
+
+        long min_age = config.getMinDaysStoring();
+        long max_age = config.getMaxDaysStoring();
+        long max_size = config.getMaxSize();
+
+        Query query = entityManager.createQuery("SELECT s FROM Stortion s ORDER BY "
+                +" (CAST(:min_age as double) + (- CAST(:max_age as double) + CAST(:min_age as double)) * POWER((CAST(s.size as double) / CAST(:max_size as double) - 1), 3)) ASC");
+        query.setParameter("min_age", (double)min_age);
+        query.setParameter("max_age", (double)max_age);
+        query.setParameter("max_size", max_size);
+
+        return (Stream<Stortion>) query.getResultStream();
+
+    }
+}

src/main/java/ru/bvn13/adastor/web/services/StortionService.java

@@ -1,15 +1,18 @@
 package ru.bvn13.adastor.web.services;
 
+import org.modelmapper.ModelMapper;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import ru.bvn13.adastor.config.Config;
 import ru.bvn13.adastor.entities.Stortion;
+import ru.bvn13.adastor.entities.dtos.StortionDto;
 import ru.bvn13.adastor.web.repositories.StortionRepository;
 
 import java.io.*;
 import java.time.LocalDateTime;
 import java.util.Optional;
 import java.util.UUID;
+import java.util.stream.Stream;
 
 /**
  * @author boykovn at 11.03.2019
@@ -19,6 +22,7 @@ public class StortionService {
 
     private StortionRepository stortionRepository;
     private Config config;
+    private ModelMapper modelMapper;
 
     @Autowired
     public void setStortionRepository(StortionRepository stortionRepository) {
@@ -30,6 +34,11 @@ public class StortionService {
         this.config = config;
     }
 
+    @Autowired
+    public void setModelMapper(ModelMapper modelMapper) {
+        this.modelMapper = modelMapper;
+    }
+
     public Optional<Stortion> findStortion(String uuid) {
         return stortionRepository.findById(uuid);
     }
@@ -45,7 +54,7 @@ public class StortionService {
         return targetStream;
     }
 
-    public Stortion createStortion(InputStream is) throws IOException {
+    public StortionDto createStortion(InputStream is) throws IOException {
         String uuid = UUID.randomUUID().toString();
         String path = String.format("/%s", uuid);
         String fullPath = String.format("%s/%s", config.getStoragePath(), uuid);
@@ -62,7 +71,28 @@ public class StortionService {
         stortion.setSize(bytesCount);
 
         stortionRepository.save(stortion);
-        return stortion;
+        return convertToDto(stortion);
+    }
+
+    private StortionDto convertToDto(Stortion stortion) {
+        StortionDto stortionDto = modelMapper.map(stortion, StortionDto.class);
+        stortionDto.setRetention(computeRetention(stortion));
+        return stortionDto;
+    }
+
+    public Stream<StortionDto> findAllSortedByRetention() {
+        Stream<Stortion> stortions = stortionRepository.findAllSortedByRetention();
+        return stortions.map(this::convertToDto);
+    }
+
+    /**
+     * RETENTION = min_age + (-max_age + min_age) * pow((file_size / max_size - 1), 3)
+     * @return retention
+     */
+    private long computeRetention(Stortion stortion) {
+        double retention = config.getMinDaysStoring()
+                + (-config.getMaxDaysStoring() + config.getMinDaysStoring()) * Math.pow((double) stortion.getSize() / config.getMaxSize() - 1, 3);
+        return Math.round(retention);
     }
 
 }

src/main/resources/application.properties

@@ -1,7 +1,13 @@
 #relative or absolute
 adastor.storage.path=./storage
-#in Megabytes
-adastor.storage.space.free=100Mb
+#in bytes
+adastor.storage.space.free=200000000
+#max stortion size in bytes
+adastor.max-size=100000000
+#min days storing
+adastor.min-days-storing=30
+#max days storing
+adastor.max-days-storing=356
 
 spring.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver
 spring.datasource.url=jdbc:hsqldb:file:./database/adastor