ThreadTracker + Attempt on NPC Disappearing fix

Engineered the ThreadTracker: server-embedded deadlock auditing tool, which will print error messages in case of found deadlocks (also showing all in-use locks on the time of the issue).
Changed the player's id on DB now starting from 20mil, thus preventing players from overwriting NPC/mobs with same oid in-game. Requires proper testing to see if the issue has been cleared.

src/tools/FilePrinter.java

@@ -44,7 +44,11 @@ public class FilePrinter {
             QUEST_UNCODED = "uncodedQuests.txt",
             AUTOSAVING_CHARACTER = "saveCharAuto.txt",
             SAVING_CHARACTER = "saveChar.txt",
-            USED_COMMANDS = "usedCommands.txt";//more to come (maps)
+            USED_COMMANDS = "usedCommands.txt",
+            DEADLOCK_ERROR = "deadlocks.txt",
+            DEADLOCK_STACK = "deadlocks/path.txt",
+            DEADLOCK_LOCKS = "deadlocks/locks.txt",
+            DEADLOCK_STATE = "deadlocks/state.txt";
     
     private static final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); //for file system purposes, it's nice to use yyyy-MM-dd
     private static final String FILE_PATH = "logs/" + sdf.format(Calendar.getInstance().getTime()) + "/"; // + sdf.format(Calendar.getInstance().getTime()) + "/"

src/tools/locks/MonitoredEnums.java

@@ -0,0 +1,81 @@
+/*
+ *     This file is part of the MapleSolaxiaV2 Maple Story Server
+ *
+ * Copyright (C) 2017 RonanLana
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package tools.locks;
+
+/**
+ *
+ * @author RonanLana
+ */
+
+public enum MonitoredEnums {
+    UNDEFINED(-1),
+    CHR(0),
+    EFF(1),
+    PET(2),
+    PRT(3),
+    CLIENT(4),
+    BOOK(5),
+    ITEM(6),
+    INVENTORY(7),
+    SHANDLER_IDLE(8),
+    SHANDLER_TEMP(9),
+    BUFF_STORAGE(10),
+    PLAYER_STORAGE(11),
+    SERVER(12),
+    MERCHANT(13),
+    CHANNEL(14),
+    GUILD(15),
+    PARTY(16),
+    WORLD_PARTY(17),
+    WORLD_OWL(18),
+    WORLD_PETS(19),
+    WORLD_MOUNTS(20),
+    WORLD_PSHOPS(21),
+    WORLD_MERCHS(21),
+    EIM(22),
+    EIM_PARTY(23),
+    EIM_SCRIPT(24),
+    EM_LOBBY(25),
+    EM_QUEUE(26),
+    CASHSHOP(27),
+    VISITOR_PSHOP(28),
+    STORAGE(29),
+    MOB_EXT(30),
+    MOB(31),
+    MOB_STATI(32),
+    MOBSKILL_FACTORY(33),
+    PORTAL(34),
+    VISITOR_MERCH(35),
+    MAP_CHRS(36),
+    MAP_OBJS(37),
+    MAP_FACTORY(38),
+    MAP_ITEM(39),
+    MINIDUNGEON(40),
+    REACTOR(41);
+    
+    private final int i;
+    
+    private MonitoredEnums(int val) {
+        this.i = val;
+    }
+
+    public int getValue() {
+        return i;
+    }
+}

src/tools/locks/MonitoredReadLock.java

@@ -0,0 +1,147 @@
+/*
+ *     This file is part of the MapleSolaxiaV2 Maple Story Server
+ *
+ * Copyright (C) 2017 RonanLana
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package tools.locks;
+
+import constants.ServerConstants;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TimeZone;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.ScheduledFuture;
+import server.TimerManager;
+import net.server.Server;
+import net.server.audit.ThreadTracker;
+
+import tools.FilePrinter;
+
+/**
+ *
+ * @author RonanLana
+ */
+public class MonitoredReadLock extends ReentrantReadWriteLock.ReadLock {
+    private ScheduledFuture<?> timeoutSchedule = null;
+    private StackTraceElement[] deadlockedState = null;
+    private final MonitoredEnums id;
+    private final int hashcode;
+    private final Lock state = new ReentrantLock(true);
+    private final AtomicInteger reentrantCount = new AtomicInteger(0);
+    
+    public MonitoredReadLock(MonitoredReentrantReadWriteLock lock) {
+        super(lock);
+        this.id = lock.id;
+        hashcode = this.hashCode();
+    }
+    
+    @Override
+    public void lock() {
+        super.lock();
+        
+        if(ServerConstants.USE_THREAD_TRACKER) {
+            if(deadlockedState != null) {
+                DateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy HH:mm:ss");
+                dateFormat.setTimeZone(TimeZone.getTimeZone(ServerConstants.TIMEZONE));
+
+                //FilePrinter.printError(FilePrinter.DEADLOCK_ERROR, "[CRITICAL] " + dateFormat.format(new Date()) + " Deadlock occurred when trying to use the '" + id.name() + "' lock resources:\r\n" + printStackTrace(deadlockedState) + "\r\n\r\n");
+                ThreadTracker.getInstance().accessThreadTracker(true, true, id, hashcode);
+                deadlockedState = null;
+            }
+
+            registerLocking();
+        }
+    }
+    
+    @Override
+    public void unlock() {
+        if(ServerConstants.USE_THREAD_TRACKER) {
+            unregisterLocking();
+        }
+        super.unlock();
+    }
+    
+    @Override
+    public boolean tryLock() {
+        if(super.tryLock()) {
+            if(ServerConstants.USE_THREAD_TRACKER) {
+                if(deadlockedState != null) {
+                    //FilePrinter.printError(FilePrinter.DEADLOCK_ERROR, "Deadlock occurred when trying to use the '" + id.name() + "' lock resources:\r\n" + printStackTrace(deadlockedState) + "\r\n\r\n");
+                    ThreadTracker.getInstance().accessThreadTracker(true, true, id, hashcode);
+                    deadlockedState = null;
+                }
+
+                registerLocking();
+            }
+            return true;
+        } else {
+            return false;
+        }
+    }
+    
+    private void registerLocking() {
+        state.lock();
+        try {
+            ThreadTracker.getInstance().accessThreadTracker(false, true, id, hashcode);
+        
+            if(reentrantCount.incrementAndGet() == 1) {
+                final Thread t = Thread.currentThread();
+                timeoutSchedule = TimerManager.getInstance().schedule(new Runnable() {
+                    @Override
+                    public void run() {
+                        issueDeadlock(t);
+                    }
+                }, ServerConstants.LOCK_MONITOR_TIME);
+            }
+        } finally {
+            state.unlock();
+        }
+    }
+    
+    private void unregisterLocking() {
+        state.lock();
+        try {
+            if(reentrantCount.decrementAndGet() == 0) {
+                if(timeoutSchedule != null) {
+                    timeoutSchedule.cancel(false);
+                    timeoutSchedule = null;
+                }
+            }
+            
+            ThreadTracker.getInstance().accessThreadTracker(false, false, id, hashcode);
+        } finally {
+            state.unlock();
+        }
+    }
+    
+    private void issueDeadlock(Thread t) {
+        deadlockedState = t.getStackTrace();
+        //super.unlock();
+    }
+    
+    private static String printStackTrace(StackTraceElement[] list) {
+        String s = "";
+        for(int i = 0; i < list.length; i++) {
+            s += ("    " + list[i].toString() + "\r\n");
+        }
+        
+        return s;
+    }
+}

src/tools/locks/MonitoredReentrantLock.java

@@ -0,0 +1,150 @@
+/*
+ *     This file is part of the MapleSolaxiaV2 Maple Story Server
+ *
+ * Copyright (C) 2017 RonanLana
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package tools.locks;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.ScheduledFuture;
+import constants.ServerConstants;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TimeZone;
+import server.TimerManager;
+import net.server.Server;
+import net.server.audit.ThreadTracker;
+import tools.FilePrinter;
+
+/**
+ *
+ * @author RonanLana
+ */
+public class MonitoredReentrantLock extends ReentrantLock {
+    private ScheduledFuture<?> timeoutSchedule = null;
+    private StackTraceElement[] deadlockedState = null;
+    private final MonitoredEnums id;
+    private final int hashcode;
+    private final Lock state = new ReentrantLock(true);
+    private final AtomicInteger reentrantCount = new AtomicInteger(0);
+   
+    public MonitoredReentrantLock(MonitoredEnums id) {
+        super();
+        this.id = id;
+        hashcode = this.hashCode();
+    }
+            
+    public MonitoredReentrantLock(MonitoredEnums id, boolean fair) {
+        super(fair);
+        this.id = id;
+        hashcode = this.hashCode();
+    }
+    
+    @Override
+    public void lock() {
+        super.lock();
+        if(ServerConstants.USE_THREAD_TRACKER) {
+            if(deadlockedState != null) {
+                DateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy HH:mm:ss");
+                dateFormat.setTimeZone(TimeZone.getTimeZone(ServerConstants.TIMEZONE));
+
+                //FilePrinter.printError(FilePrinter.DEADLOCK_ERROR, "[CRITICAL] " + dateFormat.format(new Date()) + " Deadlock occurred when trying to use the '" + id.name() + "' lock resources:\r\n" + printStackTrace(deadlockedState) + "\r\n\r\n");
+                ThreadTracker.getInstance().accessThreadTracker(true, true, id, hashcode);
+                deadlockedState = null;
+            }
+
+            registerLocking();
+        }
+    }
+    
+    @Override
+    public void unlock() {
+        if(ServerConstants.USE_THREAD_TRACKER) {
+            unregisterLocking();
+        }
+        super.unlock();
+    }
+    
+    @Override
+    public boolean tryLock() {
+        if(super.tryLock()) {
+            if(ServerConstants.USE_THREAD_TRACKER) {
+                if(deadlockedState != null) {
+                    //FilePrinter.printError(FilePrinter.DEADLOCK_ERROR, "Deadlock occurred when trying to use the '" + id.name() + "' lock resources:\r\n" + printStackTrace(deadlockedState) + "\r\n\r\n");
+                    ThreadTracker.getInstance().accessThreadTracker(true, true, id, hashcode);
+                    deadlockedState = null;
+                }
+
+                registerLocking();
+            }
+            return true;
+        } else {
+            return false;
+        }
+    }
+    
+    private void registerLocking() {
+        state.lock();
+        try {
+            ThreadTracker.getInstance().accessThreadTracker(false, true, id, hashcode);
+        
+            if(reentrantCount.incrementAndGet() == 1) {
+                final Thread t = Thread.currentThread();
+                timeoutSchedule = TimerManager.getInstance().schedule(new Runnable() {
+                    @Override
+                    public void run() {
+                        issueDeadlock(t);
+                    }
+                }, ServerConstants.LOCK_MONITOR_TIME);
+            }
+        } finally {
+            state.unlock();
+        }
+    }
+    
+    private void unregisterLocking() {
+        state.lock();
+        try {
+            if(reentrantCount.decrementAndGet() == 0) {
+                if(timeoutSchedule != null) {
+                    timeoutSchedule.cancel(false);
+                    timeoutSchedule = null;
+                }
+            }
+            
+            ThreadTracker.getInstance().accessThreadTracker(false, false, id, hashcode);
+        } finally {
+            state.unlock();
+        }
+    }
+    
+    private void issueDeadlock(Thread t) {
+        deadlockedState = t.getStackTrace();
+        //super.unlock();
+    }
+    
+    private static String printStackTrace(StackTraceElement[] list) {
+        String s = "";
+        for(int i = 0; i < list.length; i++) {
+            s += ("    " + list[i].toString() + "\r\n");
+        }
+        
+        return s;
+    }
+}

src/tools/locks/MonitoredReentrantReadWriteLock.java

@@ -0,0 +1,49 @@
+/*
+ *     This file is part of the MapleSolaxiaV2 Maple Story Server
+ *
+ * Copyright (C) 2017 RonanLana
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package tools.locks;
+
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ *
+ * @author RonanLana
+ */
+public class MonitoredReentrantReadWriteLock extends ReentrantReadWriteLock {
+    public final MonitoredEnums id;
+    
+    public MonitoredReentrantReadWriteLock(MonitoredEnums id) {
+        super();
+        this.id = id;
+    }
+            
+    public MonitoredReentrantReadWriteLock(MonitoredEnums id, boolean fair) {
+        super(fair);
+        this.id = id;
+    }
+    
+    @Override
+    public ReadLock readLock() {
+        return super.readLock();
+    }
+    
+    @Override
+    public WriteLock writeLock() {
+        return super.writeLock();
+    }
+}

src/tools/locks/MonitoredWriteLock.java

@@ -0,0 +1,145 @@
+/*
+ *     This file is part of the MapleSolaxiaV2 Maple Story Server
+ *
+ * Copyright (C) 2017 RonanLana
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package tools.locks;
+
+import constants.ServerConstants;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TimeZone;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.ScheduledFuture;
+import server.TimerManager;
+import net.server.Server;
+import net.server.audit.ThreadTracker;
+import tools.FilePrinter;
+
+/**
+ *
+ * @author RonanLana
+ */
+public class MonitoredWriteLock extends ReentrantReadWriteLock.WriteLock {
+    private ScheduledFuture<?> timeoutSchedule = null;
+    private StackTraceElement[] deadlockedState = null;
+    private final MonitoredEnums id;
+    private final int hashcode;
+    private final Lock state = new ReentrantLock(true);
+    private final AtomicInteger reentrantCount = new AtomicInteger(0);
+
+    public MonitoredWriteLock(MonitoredReentrantReadWriteLock lock) {
+        super(lock);
+        this.id = lock.id;
+        hashcode = this.hashCode();
+    }
+    
+    @Override
+    public void lock() {
+        super.lock();
+        if(ServerConstants.USE_THREAD_TRACKER) {
+            if(deadlockedState != null) {
+                DateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy HH:mm:ss");
+                dateFormat.setTimeZone(TimeZone.getTimeZone(ServerConstants.TIMEZONE));
+
+                //FilePrinter.printError(FilePrinter.DEADLOCK_ERROR, "[CRITICAL] " + dateFormat.format(new Date()) + " Deadlock occurred when trying to use the '" + id.name() + "' lock resources:\r\n" + printStackTrace(deadlockedState) + "\r\n\r\n");
+                ThreadTracker.getInstance().accessThreadTracker(true, true, id, hashcode);
+                deadlockedState = null;
+            }
+
+            registerLocking();
+        }
+    }
+    
+    @Override
+    public void unlock() {
+        if(ServerConstants.USE_THREAD_TRACKER) {
+            unregisterLocking();
+        }
+        super.unlock();
+    }
+    
+    @Override
+    public boolean tryLock() {
+        if(super.tryLock()) {
+            if(ServerConstants.USE_THREAD_TRACKER) {
+                if(deadlockedState != null) {
+                    //FilePrinter.printError(FilePrinter.DEADLOCK_ERROR, "Deadlock occurred when trying to use the '" + id.name() + "' lock resources:\r\n" + printStackTrace(deadlockedState) + "\r\n\r\n");
+                    ThreadTracker.getInstance().accessThreadTracker(true, true, id, hashcode);
+                    deadlockedState = null;
+                }
+
+                registerLocking();
+            }
+            return true;
+        } else {
+            return false;
+        }
+    }
+    
+    private void registerLocking() {
+        state.lock();
+        try {
+            ThreadTracker.getInstance().accessThreadTracker(false, true, id, hashcode);
+        
+            if(reentrantCount.incrementAndGet() == 1) {
+                final Thread t = Thread.currentThread();
+                timeoutSchedule = TimerManager.getInstance().schedule(new Runnable() {
+                    @Override
+                    public void run() {
+                        issueDeadlock(t);
+                    }
+                }, ServerConstants.LOCK_MONITOR_TIME);
+            }
+        } finally {
+            state.unlock();
+        }
+    }
+    
+    private void unregisterLocking() {
+        state.lock();
+        try {
+            if(reentrantCount.decrementAndGet() == 0) {
+                if(timeoutSchedule != null) {
+                    timeoutSchedule.cancel(false);
+                    timeoutSchedule = null;
+                }
+            }
+            
+            ThreadTracker.getInstance().accessThreadTracker(false, false, id, hashcode);
+        } finally {
+            state.unlock();
+        }
+    }
+    
+    private void issueDeadlock(Thread t) {
+        deadlockedState = t.getStackTrace();
+        //super.unlock();
+    }
+    
+    private static String printStackTrace(StackTraceElement[] list) {
+        String s = "";
+        for(int i = 0; i < list.length; i++) {
+            s += ("    " + list[i].toString() + "\r\n");
+        }
+        
+        return s;
+    }
+}