working on disk space watcher

src/main/java/ru/bvn13/adastor/config/Config.java

@@ -17,10 +17,13 @@ public class Config {
     private String storagePath;
 
     @Getter
-    // #{new Integer.parseInt('${api.orders.pingFrequency}')}
     @Value("${adastor.storage.space.free}")
     private Long freeSpace;
 
+    @Getter
+    @Value("${adastor.storage.space.critical}")
+    private Long criticalSpace;
+
     @Getter
     @Value("${adastor.max-size}")
     private Long maxSize;
@@ -47,6 +50,12 @@ public class Config {
         if (maxDaysStoring == null || maxDaysStoring.equals(0L)) {
             throw new IllegalArgumentException("Max days storing is not specified!");
         }
+        if (criticalSpace == null) {
+            throw new IllegalArgumentException("Critical space is not specified!");
+        }
+        if (criticalSpace.compareTo(freeSpace) > 0) {
+            throw new IllegalArgumentException("Critical space must be less than free space!");
+        }
     }
 
     public String getStoragePath() {

src/main/java/ru/bvn13/adastor/entities/Stortion.java

@@ -4,13 +4,10 @@ import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
-import org.springframework.beans.factory.annotation.Autowired;
-import ru.bvn13.adastor.config.Config;
 
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.Id;
-import javax.persistence.Transient;
 import java.time.LocalDateTime;
 
 /**
@@ -21,6 +18,7 @@ import java.time.LocalDateTime;
 @Setter
 @AllArgsConstructor
 @NoArgsConstructor
+// Stored portion of data :)
 public class Stortion {
 
     @Id
@@ -35,4 +33,7 @@ public class Stortion {
     @Column
     private String path;
 
+    @Column
+    private String hash;
+
 }

src/main/java/ru/bvn13/adastor/exceptions/AdastorException.java

@@ -0,0 +1,26 @@
+package ru.bvn13.adastor.exceptions;
+
+/**
+ * @author boykovn at 13.03.2019
+ */
+public class AdastorException extends Exception {
+
+    public AdastorException() {
+    }
+
+    public AdastorException(String message) {
+        super(message);
+    }
+
+    public AdastorException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public AdastorException(Throwable cause) {
+        super(cause);
+    }
+
+    public AdastorException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}

src/main/java/ru/bvn13/adastor/exceptions/InternalServerError.java

@@ -0,0 +1,23 @@
+package ru.bvn13.adastor.exceptions;
+
+/**
+ * @author boykovn at 13.03.2019
+ */
+public class InternalServerError extends AdastorException {
+
+    public InternalServerError() {
+        super();
+    }
+
+    public InternalServerError(String message) {
+        super(message);
+    }
+
+    public InternalServerError(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public InternalServerError(Throwable cause) {
+        super(cause);
+    }
+}

src/main/java/ru/bvn13/adastor/exceptions/StortionExistByHash.java

@@ -0,0 +1,16 @@
+package ru.bvn13.adastor.exceptions;
+
+import lombok.Getter;
+import ru.bvn13.adastor.entities.dtos.StortionDto;
+
+/**
+ * @author boykovn at 13.03.2019
+ */
+public class StortionExistByHash extends AdastorException {
+    @Getter
+    private StortionDto stortion;
+
+    public StortionExistByHash(StortionDto stortionDto) {
+        this.stortion = stortionDto;
+    }
+}

src/main/java/ru/bvn13/adastor/exceptions/UploadNotAvailable.java

@@ -0,0 +1,15 @@
+package ru.bvn13.adastor.exceptions;
+
+/**
+ * @author boykovn at 13.03.2019
+ */
+public class UploadNotAvailable extends AdastorException {
+
+    public UploadNotAvailable() {
+        super();
+    }
+
+    public UploadNotAvailable(String message) {
+        super(message);
+    }
+}

src/main/java/ru/bvn13/adastor/tasks/DiskFreeSpaceCheck.java

@@ -1,41 +0,0 @@
-package ru.bvn13.adastor.tasks;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.stereotype.Component;
-import ru.bvn13.adastor.config.Config;
-
-import java.io.File;
-
-/**
- * @author boykovn at 12.03.2019
- */
-@Component
-public class DiskFreeSpaceCheck {
-
-    private Config config;
-
-    @Autowired
-    public void setConfig(Config config) {
-        this.config = config;
-    }
-
-    @Scheduled(fixedDelay = 30000)
-    public void checkFreeDiskSpace() {
-        double space = getSpaceLeft();
-        if (space <= config.getFreeSpace()) {
-            removeOldStortions();
-        }
-    }
-
-    public double getSpaceLeft() {
-        File path = new File(config.getStoragePath());
-        double space = (double) path.getFreeSpace() / 1024 / 1024;
-        return space;
-    }
-
-    public void removeOldStortions() {
-
-    }
-
-}

src/main/java/ru/bvn13/adastor/tasks/DiskFreeSpaceChecker.java

@@ -0,0 +1,73 @@
+package ru.bvn13.adastor.tasks;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import ru.bvn13.adastor.config.Config;
+import ru.bvn13.adastor.web.services.StortionService;
+
+import java.io.File;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * @author boykovn at 12.03.2019
+ */
+@Component
+public class DiskFreeSpaceChecker {
+
+    private Config config;
+    private StortionService stortionService;
+
+    @Autowired
+    public void setConfig(Config config) {
+        this.config = config;
+    }
+
+    @Autowired
+    public void setStortionService(StortionService stortionService) {
+        this.stortionService = stortionService;
+    }
+
+    @Scheduled(fixedDelay = 30000)
+    public void checkFreeDiskSpace() {
+        double spaceLeft = getSpaceLeft();
+        if (spaceLeft <= config.getFreeSpace()) {
+            removeOldStortions(spaceLeft);
+        }
+    }
+
+    private double getSpaceLeft() {
+        File path = new File(config.getStoragePath());
+        double space = (double) path.getFreeSpace() / 1024 / 1024;
+        return space;
+    }
+
+    private void removeOldStortions(final double currentSpaceLeft) {
+        final double mustFreeSpace = config.getFreeSpace();
+        final AtomicReference<Double> spaceLeft = new AtomicReference<>(currentSpaceLeft);
+
+        final ExecutorService es = Executors.newFixedThreadPool(10);
+
+        stortionService.findAllSortedByRetention().forEach(stortionDto -> {
+            double space = spaceLeft.accumulateAndGet((double) stortionDto.getSize(), (a, b) -> a + b);
+            if (space >= mustFreeSpace) {
+                es.submit(() -> {
+                    File file = new File(String.format("%s%s", config.getStoragePath(), stortionDto.getPath()));
+                    if (file.exists()) {
+                        file.delete();
+                    }
+                    stortionService.removeStortionByUUID(stortionDto.getUuid());
+                });
+            }
+        });
+
+        es.shutdown();
+    }
+
+    public boolean checkUploadAvailable(long dataLength) {
+        return !(getSpaceLeft() - dataLength <= config.getCriticalSpace());
+    }
+
+}

src/main/java/ru/bvn13/adastor/web/controllers/TestController.java

@@ -4,9 +4,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.ResponseBody;
-import ru.bvn13.adastor.entities.Stortion;
 import ru.bvn13.adastor.entities.dtos.StortionDto;
-import ru.bvn13.adastor.web.repositories.StortionRepository;
 import ru.bvn13.adastor.web.services.StortionService;
 
 import java.util.stream.Stream;

src/main/java/ru/bvn13/adastor/web/controllers/UploadController.java

@@ -5,9 +5,14 @@ import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.ResponseBody;
 import ru.bvn13.adastor.entities.dtos.StortionDto;
+import ru.bvn13.adastor.exceptions.AdastorException;
+import ru.bvn13.adastor.exceptions.InternalServerError;
+import ru.bvn13.adastor.exceptions.StortionExistByHash;
+import ru.bvn13.adastor.exceptions.UploadNotAvailable;
 import ru.bvn13.adastor.web.services.StortionService;
 
 import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 import java.io.IOException;
 
 /**
@@ -25,8 +30,20 @@ public class UploadController {
 
     @PostMapping(value="/a", produces = {"application/json"})
     public @ResponseBody
-    StortionDto uploadData(HttpServletRequest request) throws IOException {
-        return stortionService.createStortion(request.getInputStream());
+    StortionDto uploadData(HttpServletRequest request, HttpServletResponse response) throws IOException, AdastorException {
+        try {
+            return stortionService.createStortion(request.getContentLengthLong(), request.getInputStream());
+        } catch (InternalServerError internalServerError) {
+            internalServerError.printStackTrace();
+            response.sendError(500, "Internal server error, Sorry");
+            return null;
+        } catch (StortionExistByHash stortionExistByHash) {
+            stortionExistByHash.printStackTrace();
+            return stortionExistByHash.getStortion();
+        } catch (UploadNotAvailable uploadNotAvailable) {
+            response.sendError(406, uploadNotAvailable.getMessage());
+            return null;
+        }
     }
 
 }

src/main/java/ru/bvn13/adastor/web/repositories/StortionRepository.java

@@ -4,10 +4,15 @@ import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.stereotype.Repository;
 import ru.bvn13.adastor.entities.Stortion;
 
+import java.util.Optional;
+
 /**
  * @author boykovn at 11.03.2019
  */
 @Repository
 public interface StortionRepository extends JpaRepository<Stortion, String>, CustomStortionRepository {
 
+    Iterable<Stortion> findAllByHash(String hash);
+    Optional<Stortion> findFirstByHash(String hash);
+
 }

src/main/java/ru/bvn13/adastor/web/services/StortionService.java

@@ -6,10 +6,19 @@ import org.springframework.stereotype.Service;
 import ru.bvn13.adastor.config.Config;
 import ru.bvn13.adastor.entities.Stortion;
 import ru.bvn13.adastor.entities.dtos.StortionDto;
+import ru.bvn13.adastor.exceptions.AdastorException;
+import ru.bvn13.adastor.exceptions.InternalServerError;
+import ru.bvn13.adastor.exceptions.StortionExistByHash;
+import ru.bvn13.adastor.exceptions.UploadNotAvailable;
+import ru.bvn13.adastor.tasks.DiskFreeSpaceChecker;
 import ru.bvn13.adastor.web.repositories.StortionRepository;
 
 import java.io.*;
+import java.security.DigestInputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.time.LocalDateTime;
+import java.util.Formatter;
 import java.util.Optional;
 import java.util.UUID;
 import java.util.stream.Stream;
@@ -23,6 +32,7 @@ public class StortionService {
     private StortionRepository stortionRepository;
     private Config config;
     private ModelMapper modelMapper;
+    private DiskFreeSpaceChecker diskFreeSpaceChecker;
 
     @Autowired
     public void setStortionRepository(StortionRepository stortionRepository) {
@@ -39,6 +49,11 @@ public class StortionService {
         this.modelMapper = modelMapper;
     }
 
+    @Autowired
+    public void setDiskFreeSpaceChecker(DiskFreeSpaceChecker diskFreeSpaceChecker) {
+        this.diskFreeSpaceChecker = diskFreeSpaceChecker;
+    }
+
     public Optional<Stortion> findStortion(String uuid) {
         return stortionRepository.findById(uuid);
     }
@@ -54,26 +69,55 @@ public class StortionService {
         return targetStream;
     }
 
-    public StortionDto createStortion(InputStream is) throws IOException {
+    public StortionDto createStortion(long dataLength, InputStream is) throws IOException, AdastorException {
+        if (!diskFreeSpaceChecker.checkUploadAvailable(dataLength)) {
+            throw new UploadNotAvailable("No space left on device!");
+        }
+
         String uuid = UUID.randomUUID().toString();
         String path = String.format("/%s", uuid);
         String fullPath = String.format("%s/%s", config.getStoragePath(), uuid);
+
+        long bytesCount;
+        String hash;
+        try(DigestInputStream dis = new DigestInputStream(new BufferedInputStream(is), MessageDigest.getInstance("SHA-1")); FileOutputStream fos = new FileOutputStream(fullPath)) {
+            bytesCount = is.transferTo(fos);
+            hash = formatMessageDigestToHex(dis);
+        } catch (NoSuchAlgorithmException e) {
+            throw new InternalServerError("SHA-1 not found, Sorry.", e);
+        }
+
+        Optional<StortionDto> similarByHash = findAnyByHash(hash);
+        if (similarByHash.isPresent()) {
+            throw new StortionExistByHash(similarByHash.get());
+        }
+
         Stortion stortion = new Stortion();
         stortion.setUuid(uuid);
         stortion.setStoreDate(LocalDateTime.now());
         stortion.setPath(path);
-
-        long bytesCount = 0;
-        try(FileOutputStream fos = new FileOutputStream(fullPath)) {
-            bytesCount = is.transferTo(fos);
-        }
-
         stortion.setSize(bytesCount);
 
         stortionRepository.save(stortion);
         return convertToDto(stortion);
     }
 
+    private String formatMessageDigestToHex(DigestInputStream dis) {
+        final MessageDigest md = dis.getMessageDigest();
+        final byte[] digest = md.digest();
+
+        // Format as HEX
+        try (Formatter formatter = new Formatter()) {
+            for (final byte b : digest) {
+                formatter.format("%02x", b);
+            }
+
+            final String sha1 = formatter.toString();
+            return sha1;
+        }
+
+    }
+
     private StortionDto convertToDto(Stortion stortion) {
         StortionDto stortionDto = modelMapper.map(stortion, StortionDto.class);
         stortionDto.setRetention(computeRetention(stortion));
@@ -95,4 +139,16 @@ public class StortionService {
         return Math.round(retention);
     }
 
+    public void removeStortionByUUID(String uuid) {
+        stortionRepository.deleteById(uuid);
+    }
+
+    private Iterable<Stortion> findAllByHash(String hash) {
+        return stortionRepository.findAllByHash(hash);
+    }
+
+    private Optional<StortionDto> findAnyByHash(String hash) {
+        return stortionRepository.findFirstByHash(hash).map(this::convertToDto);
+    }
+
 }

src/main/resources/application.properties

@@ -2,6 +2,7 @@
 adastor.storage.path=./storage
 #in bytes
 adastor.storage.space.free=200000000
+adastor.storage.space.critical=10000000
 #max stortion size in bytes
 adastor.max-size=100000000
 #min days storing