Merge remote-tracking branch 'origin/bloomfilters' into bloomfilters

src/main/java/ru/vk/itmo/dyagayalexandra/CompactManager.java

@@ -0,0 +1,166 @@
+package ru.vk.itmo.dyagayalexandra;
+
+import ru.vk.itmo.Entry;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
+
+public class CompactManager {
+
+    private final String fileName;
+    private final String fileExtension;
+    private final String fileIndexName;
+    private Path compactedFile;
+    private Path compactedIndex;
+    private final EntryKeyComparator entryKeyComparator;
+    private final FileWriterManager fileWriterManager;
+    private static final String TEMP_FILE_NAME = "tempCompact.txt";
+    private static final String TEMP_FILE_INDEX_NAME = "tempCompactIndex.txt";
+    private static final String MAIN_FILE_NAME = "mainCompact.txt";
+    private static final String MAIN_FILE_INDEX_NAME = "mainCompactIndex.txt";
+
+    public CompactManager(String fileName, String fileIndexName, String fileExtension,
+                          EntryKeyComparator entryKeyComparator, FileWriterManager fileWriterManager) {
+        this.fileName = fileName;
+        this.fileIndexName = fileIndexName;
+        this.fileExtension = fileExtension;
+        this.entryKeyComparator = entryKeyComparator;
+        this.fileWriterManager = fileWriterManager;
+    }
+
+    void compact(Path basePath, FileManager fileManager) throws IOException {
+        long indexOffset = Long.BYTES;
+        long offset;
+        compactedFile = basePath.resolve(TEMP_FILE_NAME);
+        compactedIndex = basePath.resolve(TEMP_FILE_INDEX_NAME);
+
+        Iterator<Entry<MemorySegment>> iterator =
+                MergedIterator.createMergedIterator(fileManager.createIterators(null, null), entryKeyComparator);
+
+        long storageSize = 0;
+        long tableSize = 0;
+        while (iterator.hasNext()) {
+            Entry<MemorySegment> currentItem = iterator.next();
+            tableSize += 2 * Integer.BYTES + currentItem.key().byteSize();
+            if (currentItem.value() != null) {
+                tableSize += currentItem.value().byteSize();
+            }
+
+            storageSize++;
+        }
+
+        iterator = MergedIterator.createMergedIterator(fileManager.createIterators(null, null), entryKeyComparator);
+        long tableOffset = 0;
+
+        try (FileChannel tableChannel = FileChannel.open(compactedFile, StandardOpenOption.READ,
+                StandardOpenOption.WRITE, StandardOpenOption.CREATE);
+             FileChannel indexChannel = FileChannel.open(compactedIndex, StandardOpenOption.READ,
+                     StandardOpenOption.WRITE, StandardOpenOption.CREATE);
+             Arena arena = Arena.ofConfined()) {
+            MemorySegment tableMemorySegment = tableChannel.map(FileChannel.MapMode.READ_WRITE,
+                    0, tableSize, arena);
+            MemorySegment indexMemorySegment = indexChannel.map(FileChannel.MapMode.READ_WRITE,
+                    0, (storageSize + 1) * Long.BYTES, arena);
+            fileWriterManager.writeStorageSize(indexMemorySegment, storageSize);
+            while (iterator.hasNext()) {
+                Entry<MemorySegment> currentItem = iterator.next();
+                offset = fileWriterManager.writeEntry(tableMemorySegment, tableOffset, currentItem);
+                fileWriterManager.writeIndexes(indexMemorySegment, indexOffset, tableOffset);
+                tableOffset = offset;
+                indexOffset += Long.BYTES;
+            }
+        }
+
+        Files.move(compactedFile, basePath.resolve(MAIN_FILE_NAME), ATOMIC_MOVE);
+        Files.move(compactedIndex, basePath.resolve(MAIN_FILE_INDEX_NAME), ATOMIC_MOVE);
+        compactedFile = basePath.resolve(MAIN_FILE_NAME);
+        compactedIndex = basePath.resolve(MAIN_FILE_INDEX_NAME);
+    }
+
+    void renameCompactFile(Path basePath) throws IOException {
+        Files.move(compactedFile, basePath.resolve(fileName + "0" + fileExtension), ATOMIC_MOVE);
+        Files.move(compactedIndex, basePath.resolve(fileIndexName + "0" + fileExtension), ATOMIC_MOVE);
+    }
+
+    void deleteAllFiles(List<Path> ssTables, List<Path> ssIndexes) {
+        for (Path ssTable : ssTables) {
+            try {
+                Files.deleteIfExists(ssTable);
+            } catch (IOException e) {
+                throw new UncheckedIOException("Error deleting a ssTable file.", e);
+            }
+        }
+
+        for (Path ssIndex : ssIndexes) {
+            try {
+                Files.deleteIfExists(ssIndex);
+            } catch (IOException e) {
+                throw new UncheckedIOException("Error deleting a ssIndex file.", e);
+            }
+        }
+    }
+
+    boolean deleteTempFile(Path basePath, Path file, List<Path> files) throws IOException {
+        if (file.equals(basePath.resolve(TEMP_FILE_NAME))) {
+            Files.delete(basePath.resolve(TEMP_FILE_NAME));
+            if (files.contains(basePath.resolve(TEMP_FILE_INDEX_NAME))) {
+                Files.delete(basePath.resolve(TEMP_FILE_INDEX_NAME));
+            }
+
+            return true;
+        }
+
+        return false;
+    }
+
+    boolean clearIfCompactFileExists(Path basePath, Path file, List<Path> files) throws IOException {
+        if (file.equals(basePath.resolve(MAIN_FILE_NAME))) {
+            if (files.contains(basePath.resolve(MAIN_FILE_INDEX_NAME))) {
+                List<Path> ssTables = new ArrayList<>();
+                List<Path> ssIndexes = new ArrayList<>();
+                for (Path currentFile : files) {
+                    if (currentFile.getFileName().toString().startsWith(fileName)) {
+                        ssTables.add(currentFile);
+                    }
+
+                    if (currentFile.getFileName().toString().startsWith(fileIndexName)) {
+                        ssIndexes.add(currentFile);
+                    }
+                }
+
+                ssTables.sort(new PathsComparator(fileName, fileExtension));
+                ssIndexes.sort(new PathsComparator(fileIndexName, fileExtension));
+                deleteFiles(ssTables);
+                deleteFiles(ssIndexes);
+
+                Files.move(basePath.resolve(MAIN_FILE_NAME),
+                        basePath.resolve(fileName + "0" + fileExtension), ATOMIC_MOVE);
+                Files.move(basePath.resolve(MAIN_FILE_INDEX_NAME),
+                        basePath.resolve(fileIndexName + "0" + fileExtension), ATOMIC_MOVE);
+            } else {
+                Files.delete(basePath.resolve(MAIN_FILE_NAME));
+            }
+
+            return true;
+        }
+
+        return false;
+    }
+
+    private void deleteFiles(List<Path> filePaths) throws IOException {
+        for (Path filePath : filePaths) {
+            Files.delete(filePath);
+        }
+    }
+}

src/main/java/ru/vk/itmo/dyagayalexandra/EntryKeyComparator.java

@@ -7,7 +7,11 @@
 
 public class EntryKeyComparator implements Comparator<Entry<MemorySegment>> {
 
-    private final MemorySegmentComparator memorySegmentComparator = new MemorySegmentComparator();
+    private final MemorySegmentComparator memorySegmentComparator;
+
+    public EntryKeyComparator(MemorySegmentComparator memorySegmentComparator) {
+        this.memorySegmentComparator = memorySegmentComparator;
+    }
 
     @Override
     public int compare(Entry<MemorySegment> entry1, Entry<MemorySegment> entry2) {

src/main/java/ru/vk/itmo/dyagayalexandra/FileChecker.java

@@ -0,0 +1,138 @@
+package ru.vk.itmo.dyagayalexandra;
+
+import java.io.IOException;
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
+import java.nio.channels.FileChannel;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class FileChecker {
+
+    private final String fileName;
+    private final String fileExtension;
+    private final String fileIndexName;
+    private final CompactManager compactManager;
+
+    public FileChecker(String fileName, String fileIndexName, String fileExtension, CompactManager compactManager) {
+        this.fileName = fileName;
+        this.fileIndexName = fileIndexName;
+        this.fileExtension = fileExtension;
+        this.compactManager = compactManager;
+    }
+
+    public List<Map.Entry<MemorySegment, MemorySegment>> checkFiles(Path basePath, Arena arena) throws IOException {
+        List<Map.Entry<MemorySegment, MemorySegment>> allFiles = new ArrayList<>();
+        List<Path> ssTablesPaths = new ArrayList<>();
+        List<Path> ssIndexesPaths = new ArrayList<>();
+
+        List<Path> files = getAllFiles(basePath);
+        checkFile(basePath, files);
+
+        List<Map.Entry<Path, Path>> allDataPaths = getAllDataPaths(basePath);
+
+        for (Map.Entry<Path, Path> entry : allDataPaths) {
+            ssTablesPaths.add(entry.getKey());
+            ssIndexesPaths.add(entry.getValue());
+        }
+
+        ssTablesPaths.sort(new PathsComparator(fileName, fileExtension));
+        ssIndexesPaths.sort(new PathsComparator(fileIndexName, fileExtension));
+
+        if (ssTablesPaths.size() != ssIndexesPaths.size()) {
+            throw new NoSuchFileException("Not all files found.");
+        }
+
+        for (int i = 0; i < ssTablesPaths.size(); i++) {
+            MemorySegment data;
+            MemorySegment index;
+            Path dataPath = ssTablesPaths.get(i);
+            Path indexPath = ssIndexesPaths.get(i);
+
+            try (FileChannel fileChannel = FileChannel.open(dataPath)) {
+                data = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, fileChannel.size(), arena);
+            }
+
+            try (FileChannel fileChannel = FileChannel.open(indexPath)) {
+                index = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, fileChannel.size(), arena);
+            }
+
+            checkFileMatch(dataPath, indexPath);
+            allFiles.add(new AbstractMap.SimpleEntry<>(data, index));
+        }
+
+        return allFiles;
+    }
+
+    public List<Map.Entry<Path, Path>> getAllDataPaths(Path basePath) throws IOException {
+        List<Path> files = getAllFiles(basePath);
+        List<Path> ssTablesPaths = new ArrayList<>();
+        List<Path> ssIndexesPaths = new ArrayList<>();
+        List<Map.Entry<Path, Path>> filePathsMap = new ArrayList<>();
+        for (Path file : files) {
+            if (String.valueOf(file.getFileName()).startsWith(fileName)) {
+                ssTablesPaths.add(file);
+            }
+
+            if (String.valueOf(file.getFileName()).startsWith(fileIndexName)) {
+                ssIndexesPaths.add(file);
+            }
+        }
+
+        ssTablesPaths.sort(new PathsComparator(fileName, fileExtension));
+        ssIndexesPaths.sort(new PathsComparator(fileIndexName, fileExtension));
+
+        int size = ssTablesPaths.size();
+        for (int i = 0; i < size; i++) {
+            filePathsMap.add(new AbstractMap.SimpleEntry<>(ssTablesPaths.get(i), ssIndexesPaths.get(i)));
+        }
+
+        return filePathsMap;
+    }
+
+    private void checkFile(Path basePath, List<Path> files) throws IOException {
+        for (Path file : files) {
+            if (compactManager.deleteTempFile(basePath, file, files)) {
+                break;
+            }
+
+            if (compactManager.clearIfCompactFileExists(basePath, file, files)) {
+                break;
+            }
+        }
+    }
+
+    private void checkFileMatch(Path data, Path index) throws IOException {
+        String dataString = data.toString();
+        String indexString = index.toString();
+        if (Integer.parseInt(dataString.substring(
+                dataString.indexOf(fileName) + fileName.length(),
+                dataString.indexOf(fileExtension)))
+                != Integer.parseInt(indexString.substring(
+                indexString.indexOf(fileIndexName) + fileIndexName.length(),
+                indexString.indexOf(fileExtension)))) {
+            throw new NoSuchFileException("The files don't match.");
+        }
+    }
+
+    private List<Path> getAllFiles(Path basePath) throws IOException {
+        List<Path> files = new ArrayList<>();
+        if (!Files.exists(basePath)) {
+            Files.createDirectory(basePath);
+        }
+
+        try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(basePath)) {
+            for (Path path : directoryStream) {
+                files.add(path);
+            }
+        }
+
+        return files;
+    }
+}

src/main/java/ru/vk/itmo/dyagayalexandra/FileIterator.java

@@ -13,13 +13,15 @@ public class FileIterator implements Iterator<Entry<MemorySegment>> {
     private final MemorySegment ssIndex;
     private long currentIndex;
     private final long endIndex;
+    private final FileReaderManager fileReaderManager;
 
     public FileIterator(MemorySegment ssTable, MemorySegment ssIndex, MemorySegment from,
-                        MemorySegment to, long indexSize) throws IOException {
+                        MemorySegment to, long indexSize, FileReaderManager fileReaderManager) throws IOException {
         this.ssTable = ssTable;
         this.ssIndex = ssIndex;
-        currentIndex = from == null ? 0 : FileManager.getEntryIndex(ssTable, ssIndex, from, indexSize);
-        endIndex = to == null ? indexSize : FileManager.getEntryIndex(ssTable, ssIndex, to, indexSize);
+        currentIndex = from == null ? 0 : fileReaderManager.getEntryIndex(ssTable, ssIndex, from, indexSize);
+        endIndex = to == null ? indexSize : fileReaderManager.getEntryIndex(ssTable, ssIndex, to, indexSize);
+        this.fileReaderManager = fileReaderManager;
     }
 
     @Override
@@ -35,10 +37,11 @@ public Entry<MemorySegment> next() {
 
         Entry<MemorySegment> entry;
         try {
-            entry = FileManager.getCurrentEntry(currentIndex, ssTable, ssIndex);
+            entry = fileReaderManager.getCurrentEntry(currentIndex, ssTable, ssIndex);
         } catch (IOException e) {
             throw new NoSuchElementException("There is no next element.", e);
         }
+
         currentIndex++;
         return entry;
     }