diff --git a/src/main/java/tools/mapletools/RemoveDuplicateDrops.java b/src/main/java/tools/mapletools/RemoveDuplicateDrops.java new file mode 100644 index 0000000000..03f7d7fc6f --- /dev/null +++ b/src/main/java/tools/mapletools/RemoveDuplicateDrops.java @@ -0,0 +1,122 @@ +package tools.mapletools; + +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * This tool takes the large drop-data.sql file as input, and removes all duplicate drops. + * A drop is considered to be a duplicate if it has the same dropperid/mobid & itemid as another drop. + * The first drop encountered in the input is kept, and any subsequent duplicates are removed. + *

+ * Note about duplicate drops: duplicate drops are supposed to exist. + * For example, Zakum should be able to drop multiple Zakum helmets. This is vanilla MapleStory behavior. + *

+ * This tool is mostly useful during the migration from the old scripts to the new scripts in Liquibase, + * since the old scripts relied on a combination of unique database constraint on mobid+itemid + * and the INSERT IGNORE feature of MySQL to prevent duplicates. The INSERT IGNORE would attempt to insert the drop, + * and if there was a duplicate it would fail and thereby be skipped. + * In the new scripts, the unique constraint has been removed (because duplicate drops should be allowed), + * so all the (previously ignored) inserts would succeed, and we end up with a bunch of duplicates. + * + * @author Ponk + */ +public class RemoveDuplicateDrops { + // Precondition: copy from src/main/resources/db/(...)drop-data.sql to tools/input/drop-data.sql + private static final Path DROP_DATA_INPUT_FILE = ToolConstants.getInputFile("drop-data.sql"); + private static final Path DROP_DATA_OUTPUT_FILE = ToolConstants.getOutputFile("drop-data_no-duplicates.sql"); + private static final Path REMOVED_LINES_OUTPUT_FILE = ToolConstants.getOutputFile("drop-data_removed-lines.sql"); + private static final Pattern INSERT_VALUE_PATTERN = Pattern.compile(".*\\((?\\d+), (?\\d+), \\d+, \\d+, \\d+, \\d+\\).*"); + + private record DropIdentifier(int mobId, int itemId) { + } + + private record ProcessingResult(List retainedLines, List removedLines) { + } + + public static void main(String[] args) { + Instant start = Instant.now(); + + System.out.printf("Reading %s%n", DROP_DATA_INPUT_FILE); + List lines = readDropDataLines(); + System.out.printf("Read %d lines%n", lines.size()); + + System.out.println("Removing duplicate drops..."); + ProcessingResult processingResult = removeDuplicateDrops(lines); + System.out.printf("Removed %d lines%n", processingResult.removedLines.size()); + + System.out.println("Writing output to " + DROP_DATA_OUTPUT_FILE); + writeDropDataOutput(processingResult.retainedLines); + System.out.println("Writing removed lines to " + REMOVED_LINES_OUTPUT_FILE); + writeRemovedLinesOutput(processingResult.removedLines); + + Duration totalDuration = Duration.between(start, Instant.now()); + System.out.printf("Done! Total elapsed time: %d%n", totalDuration.toMillis()); + } + + private static List readDropDataLines() { + try { + return Files.readAllLines(DROP_DATA_INPUT_FILE); + } catch (Exception e) { + throw new RuntimeException("Failed to read input file", e); + } + } + + private static ProcessingResult removeDuplicateDrops(List lines) { + Set encounteredDrops = new HashSet<>(); + List retainedLines = new ArrayList<>(); + List removedLines = new ArrayList<>(); + for (String line : lines) { + Optional optDropIdentifier = parseDropIdentifier(line); + if (optDropIdentifier.isEmpty()) { + retainedLines.add(line); + continue; + } + + DropIdentifier dropIdentifier = optDropIdentifier.get(); + if (encounteredDrops.contains(dropIdentifier)) { + removedLines.add(line); + } else { + encounteredDrops.add(dropIdentifier); + retainedLines.add(line); + } + } + + return new ProcessingResult(retainedLines, removedLines); + } + + private static Optional parseDropIdentifier(String line) { + Matcher matcher = INSERT_VALUE_PATTERN.matcher(line); + if (!matcher.matches()) { + return Optional.empty(); + } + int mobId = Integer.parseInt(matcher.group("mob")); + int itemId = Integer.parseInt(matcher.group("item")); + return Optional.of(new DropIdentifier(mobId, itemId)); + } + + private static void writeDropDataOutput(List retainedLines) { + try (PrintWriter pw = new PrintWriter(Files.newOutputStream(DROP_DATA_OUTPUT_FILE))) { + retainedLines.forEach(pw::println); + } catch (Exception e) { + throw new RuntimeException("Failed to write drop data output file", e); + } + } + + private static void writeRemovedLinesOutput(List removedLines) { + try (PrintWriter pw = new PrintWriter(Files.newOutputStream(REMOVED_LINES_OUTPUT_FILE))) { + removedLines.forEach(pw::println); + } catch (Exception e) { + throw new RuntimeException("Failed to write removed lines output file", e); + } + } +}