diff --git a/src/main/java/ru/bvn13/adastor/config/Config.java b/src/main/java/ru/bvn13/adastor/config/Config.java index 9962d31..fd2c5e9 100644 --- a/src/main/java/ru/bvn13/adastor/config/Config.java +++ b/src/main/java/ru/bvn13/adastor/config/Config.java @@ -17,10 +17,13 @@ public class Config { private String storagePath; @Getter - // #{new Integer.parseInt('${api.orders.pingFrequency}')} @Value("${adastor.storage.space.free}") private Long freeSpace; + @Getter + @Value("${adastor.storage.space.critical}") + private Long criticalSpace; + @Getter @Value("${adastor.max-size}") private Long maxSize; @@ -47,6 +50,12 @@ public class Config { if (maxDaysStoring == null || maxDaysStoring.equals(0L)) { throw new IllegalArgumentException("Max days storing is not specified!"); } + if (criticalSpace == null) { + throw new IllegalArgumentException("Critical space is not specified!"); + } + if (criticalSpace.compareTo(freeSpace) > 0) { + throw new IllegalArgumentException("Critical space must be less than free space!"); + } } public String getStoragePath() { diff --git a/src/main/java/ru/bvn13/adastor/entities/Stortion.java b/src/main/java/ru/bvn13/adastor/entities/Stortion.java index a864245..b488506 100644 --- a/src/main/java/ru/bvn13/adastor/entities/Stortion.java +++ b/src/main/java/ru/bvn13/adastor/entities/Stortion.java @@ -4,13 +4,10 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; -import org.springframework.beans.factory.annotation.Autowired; -import ru.bvn13.adastor.config.Config; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Id; -import javax.persistence.Transient; import java.time.LocalDateTime; /** @@ -21,6 +18,7 @@ import java.time.LocalDateTime; @Setter @AllArgsConstructor @NoArgsConstructor +// Stored portion of data :) public class Stortion { @Id @@ -35,4 +33,7 @@ public class Stortion { @Column private String path; + @Column + private String hash; + } diff --git a/src/main/java/ru/bvn13/adastor/exceptions/AdastorException.java b/src/main/java/ru/bvn13/adastor/exceptions/AdastorException.java new file mode 100644 index 0000000..9c4919b --- /dev/null +++ b/src/main/java/ru/bvn13/adastor/exceptions/AdastorException.java @@ -0,0 +1,26 @@ +package ru.bvn13.adastor.exceptions; + +/** + * @author boykovn at 13.03.2019 + */ +public class AdastorException extends Exception { + + public AdastorException() { + } + + public AdastorException(String message) { + super(message); + } + + public AdastorException(String message, Throwable cause) { + super(message, cause); + } + + public AdastorException(Throwable cause) { + super(cause); + } + + public AdastorException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/src/main/java/ru/bvn13/adastor/exceptions/InternalServerError.java b/src/main/java/ru/bvn13/adastor/exceptions/InternalServerError.java new file mode 100644 index 0000000..ecf54d7 --- /dev/null +++ b/src/main/java/ru/bvn13/adastor/exceptions/InternalServerError.java @@ -0,0 +1,23 @@ +package ru.bvn13.adastor.exceptions; + +/** + * @author boykovn at 13.03.2019 + */ +public class InternalServerError extends AdastorException { + + public InternalServerError() { + super(); + } + + public InternalServerError(String message) { + super(message); + } + + public InternalServerError(String message, Throwable cause) { + super(message, cause); + } + + public InternalServerError(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/ru/bvn13/adastor/exceptions/StortionExistByHash.java b/src/main/java/ru/bvn13/adastor/exceptions/StortionExistByHash.java new file mode 100644 index 0000000..a87ff0a --- /dev/null +++ b/src/main/java/ru/bvn13/adastor/exceptions/StortionExistByHash.java @@ -0,0 +1,16 @@ +package ru.bvn13.adastor.exceptions; + +import lombok.Getter; +import ru.bvn13.adastor.entities.dtos.StortionDto; + +/** + * @author boykovn at 13.03.2019 + */ +public class StortionExistByHash extends AdastorException { + @Getter + private StortionDto stortion; + + public StortionExistByHash(StortionDto stortionDto) { + this.stortion = stortionDto; + } +} diff --git a/src/main/java/ru/bvn13/adastor/exceptions/UploadNotAvailable.java b/src/main/java/ru/bvn13/adastor/exceptions/UploadNotAvailable.java new file mode 100644 index 0000000..5e5067f --- /dev/null +++ b/src/main/java/ru/bvn13/adastor/exceptions/UploadNotAvailable.java @@ -0,0 +1,15 @@ +package ru.bvn13.adastor.exceptions; + +/** + * @author boykovn at 13.03.2019 + */ +public class UploadNotAvailable extends AdastorException { + + public UploadNotAvailable() { + super(); + } + + public UploadNotAvailable(String message) { + super(message); + } +} diff --git a/src/main/java/ru/bvn13/adastor/tasks/DiskFreeSpaceCheck.java b/src/main/java/ru/bvn13/adastor/tasks/DiskFreeSpaceCheck.java deleted file mode 100644 index f6a632c..0000000 --- a/src/main/java/ru/bvn13/adastor/tasks/DiskFreeSpaceCheck.java +++ /dev/null @@ -1,41 +0,0 @@ -package ru.bvn13.adastor.tasks; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Component; -import ru.bvn13.adastor.config.Config; - -import java.io.File; - -/** - * @author boykovn at 12.03.2019 - */ -@Component -public class DiskFreeSpaceCheck { - - private Config config; - - @Autowired - public void setConfig(Config config) { - this.config = config; - } - - @Scheduled(fixedDelay = 30000) - public void checkFreeDiskSpace() { - double space = getSpaceLeft(); - if (space <= config.getFreeSpace()) { - removeOldStortions(); - } - } - - public double getSpaceLeft() { - File path = new File(config.getStoragePath()); - double space = (double) path.getFreeSpace() / 1024 / 1024; - return space; - } - - public void removeOldStortions() { - - } - -} diff --git a/src/main/java/ru/bvn13/adastor/tasks/DiskFreeSpaceChecker.java b/src/main/java/ru/bvn13/adastor/tasks/DiskFreeSpaceChecker.java new file mode 100644 index 0000000..c341032 --- /dev/null +++ b/src/main/java/ru/bvn13/adastor/tasks/DiskFreeSpaceChecker.java @@ -0,0 +1,73 @@ +package ru.bvn13.adastor.tasks; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import ru.bvn13.adastor.config.Config; +import ru.bvn13.adastor.web.services.StortionService; + +import java.io.File; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicReference; + +/** + * @author boykovn at 12.03.2019 + */ +@Component +public class DiskFreeSpaceChecker { + + private Config config; + private StortionService stortionService; + + @Autowired + public void setConfig(Config config) { + this.config = config; + } + + @Autowired + public void setStortionService(StortionService stortionService) { + this.stortionService = stortionService; + } + + @Scheduled(fixedDelay = 30000) + public void checkFreeDiskSpace() { + double spaceLeft = getSpaceLeft(); + if (spaceLeft <= config.getFreeSpace()) { + removeOldStortions(spaceLeft); + } + } + + private double getSpaceLeft() { + File path = new File(config.getStoragePath()); + double space = (double) path.getFreeSpace() / 1024 / 1024; + return space; + } + + private void removeOldStortions(final double currentSpaceLeft) { + final double mustFreeSpace = config.getFreeSpace(); + final AtomicReference spaceLeft = new AtomicReference<>(currentSpaceLeft); + + final ExecutorService es = Executors.newFixedThreadPool(10); + + stortionService.findAllSortedByRetention().forEach(stortionDto -> { + double space = spaceLeft.accumulateAndGet((double) stortionDto.getSize(), (a, b) -> a + b); + if (space >= mustFreeSpace) { + es.submit(() -> { + File file = new File(String.format("%s%s", config.getStoragePath(), stortionDto.getPath())); + if (file.exists()) { + file.delete(); + } + stortionService.removeStortionByUUID(stortionDto.getUuid()); + }); + } + }); + + es.shutdown(); + } + + public boolean checkUploadAvailable(long dataLength) { + return !(getSpaceLeft() - dataLength <= config.getCriticalSpace()); + } + +} diff --git a/src/main/java/ru/bvn13/adastor/web/controllers/TestController.java b/src/main/java/ru/bvn13/adastor/web/controllers/TestController.java index e55a7cf..b3dcd4b 100644 --- a/src/main/java/ru/bvn13/adastor/web/controllers/TestController.java +++ b/src/main/java/ru/bvn13/adastor/web/controllers/TestController.java @@ -4,9 +4,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.ResponseBody; -import ru.bvn13.adastor.entities.Stortion; import ru.bvn13.adastor.entities.dtos.StortionDto; -import ru.bvn13.adastor.web.repositories.StortionRepository; import ru.bvn13.adastor.web.services.StortionService; import java.util.stream.Stream; diff --git a/src/main/java/ru/bvn13/adastor/web/controllers/UploadController.java b/src/main/java/ru/bvn13/adastor/web/controllers/UploadController.java index 9788287..75e8770 100644 --- a/src/main/java/ru/bvn13/adastor/web/controllers/UploadController.java +++ b/src/main/java/ru/bvn13/adastor/web/controllers/UploadController.java @@ -5,9 +5,14 @@ import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.ResponseBody; import ru.bvn13.adastor.entities.dtos.StortionDto; +import ru.bvn13.adastor.exceptions.AdastorException; +import ru.bvn13.adastor.exceptions.InternalServerError; +import ru.bvn13.adastor.exceptions.StortionExistByHash; +import ru.bvn13.adastor.exceptions.UploadNotAvailable; import ru.bvn13.adastor.web.services.StortionService; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.io.IOException; /** @@ -25,8 +30,20 @@ public class UploadController { @PostMapping(value="/a", produces = {"application/json"}) public @ResponseBody - StortionDto uploadData(HttpServletRequest request) throws IOException { - return stortionService.createStortion(request.getInputStream()); + StortionDto uploadData(HttpServletRequest request, HttpServletResponse response) throws IOException, AdastorException { + try { + return stortionService.createStortion(request.getContentLengthLong(), request.getInputStream()); + } catch (InternalServerError internalServerError) { + internalServerError.printStackTrace(); + response.sendError(500, "Internal server error, Sorry"); + return null; + } catch (StortionExistByHash stortionExistByHash) { + stortionExistByHash.printStackTrace(); + return stortionExistByHash.getStortion(); + } catch (UploadNotAvailable uploadNotAvailable) { + response.sendError(406, uploadNotAvailable.getMessage()); + return null; + } } } diff --git a/src/main/java/ru/bvn13/adastor/web/repositories/StortionRepository.java b/src/main/java/ru/bvn13/adastor/web/repositories/StortionRepository.java index c13e6cf..10a967a 100644 --- a/src/main/java/ru/bvn13/adastor/web/repositories/StortionRepository.java +++ b/src/main/java/ru/bvn13/adastor/web/repositories/StortionRepository.java @@ -4,10 +4,15 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; import ru.bvn13.adastor.entities.Stortion; +import java.util.Optional; + /** * @author boykovn at 11.03.2019 */ @Repository public interface StortionRepository extends JpaRepository, CustomStortionRepository { + Iterable findAllByHash(String hash); + Optional findFirstByHash(String hash); + } diff --git a/src/main/java/ru/bvn13/adastor/web/services/StortionService.java b/src/main/java/ru/bvn13/adastor/web/services/StortionService.java index 189b4b3..baf0123 100644 --- a/src/main/java/ru/bvn13/adastor/web/services/StortionService.java +++ b/src/main/java/ru/bvn13/adastor/web/services/StortionService.java @@ -6,10 +6,19 @@ import org.springframework.stereotype.Service; import ru.bvn13.adastor.config.Config; import ru.bvn13.adastor.entities.Stortion; import ru.bvn13.adastor.entities.dtos.StortionDto; +import ru.bvn13.adastor.exceptions.AdastorException; +import ru.bvn13.adastor.exceptions.InternalServerError; +import ru.bvn13.adastor.exceptions.StortionExistByHash; +import ru.bvn13.adastor.exceptions.UploadNotAvailable; +import ru.bvn13.adastor.tasks.DiskFreeSpaceChecker; import ru.bvn13.adastor.web.repositories.StortionRepository; import java.io.*; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.time.LocalDateTime; +import java.util.Formatter; import java.util.Optional; import java.util.UUID; import java.util.stream.Stream; @@ -23,6 +32,7 @@ public class StortionService { private StortionRepository stortionRepository; private Config config; private ModelMapper modelMapper; + private DiskFreeSpaceChecker diskFreeSpaceChecker; @Autowired public void setStortionRepository(StortionRepository stortionRepository) { @@ -39,6 +49,11 @@ public class StortionService { this.modelMapper = modelMapper; } + @Autowired + public void setDiskFreeSpaceChecker(DiskFreeSpaceChecker diskFreeSpaceChecker) { + this.diskFreeSpaceChecker = diskFreeSpaceChecker; + } + public Optional findStortion(String uuid) { return stortionRepository.findById(uuid); } @@ -54,26 +69,55 @@ public class StortionService { return targetStream; } - public StortionDto createStortion(InputStream is) throws IOException { + public StortionDto createStortion(long dataLength, InputStream is) throws IOException, AdastorException { + if (!diskFreeSpaceChecker.checkUploadAvailable(dataLength)) { + throw new UploadNotAvailable("No space left on device!"); + } + String uuid = UUID.randomUUID().toString(); String path = String.format("/%s", uuid); String fullPath = String.format("%s/%s", config.getStoragePath(), uuid); + + long bytesCount; + String hash; + try(DigestInputStream dis = new DigestInputStream(new BufferedInputStream(is), MessageDigest.getInstance("SHA-1")); FileOutputStream fos = new FileOutputStream(fullPath)) { + bytesCount = is.transferTo(fos); + hash = formatMessageDigestToHex(dis); + } catch (NoSuchAlgorithmException e) { + throw new InternalServerError("SHA-1 not found, Sorry.", e); + } + + Optional similarByHash = findAnyByHash(hash); + if (similarByHash.isPresent()) { + throw new StortionExistByHash(similarByHash.get()); + } + Stortion stortion = new Stortion(); stortion.setUuid(uuid); stortion.setStoreDate(LocalDateTime.now()); stortion.setPath(path); - - long bytesCount = 0; - try(FileOutputStream fos = new FileOutputStream(fullPath)) { - bytesCount = is.transferTo(fos); - } - stortion.setSize(bytesCount); stortionRepository.save(stortion); return convertToDto(stortion); } + private String formatMessageDigestToHex(DigestInputStream dis) { + final MessageDigest md = dis.getMessageDigest(); + final byte[] digest = md.digest(); + + // Format as HEX + try (Formatter formatter = new Formatter()) { + for (final byte b : digest) { + formatter.format("%02x", b); + } + + final String sha1 = formatter.toString(); + return sha1; + } + + } + private StortionDto convertToDto(Stortion stortion) { StortionDto stortionDto = modelMapper.map(stortion, StortionDto.class); stortionDto.setRetention(computeRetention(stortion)); @@ -95,4 +139,16 @@ public class StortionService { return Math.round(retention); } + public void removeStortionByUUID(String uuid) { + stortionRepository.deleteById(uuid); + } + + private Iterable findAllByHash(String hash) { + return stortionRepository.findAllByHash(hash); + } + + private Optional findAnyByHash(String hash) { + return stortionRepository.findFirstByHash(hash).map(this::convertToDto); + } + } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 295fe16..f2c4e03 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -2,6 +2,7 @@ adastor.storage.path=./storage #in bytes adastor.storage.space.free=200000000 +adastor.storage.space.critical=10000000 #max stortion size in bytes adastor.max-size=100000000 #min days storing