ThreadTracker + Attempt on NPC Disappearing fix
Engineered the ThreadTracker: server-embedded deadlock auditing tool, which will print error messages in case of found deadlocks (also showing all in-use locks on the time of the issue). Changed the player's id on DB now starting from 20mil, thus preventing players from overwriting NPC/mobs with same oid in-game. Requires proper testing to see if the issue has been cleared.
This commit is contained in:
ronancpl
281
src/net/server/audit/ThreadTracker.java
Normal file
281
src/net/server/audit/ThreadTracker.java
Normal file
@@ -0,0 +1,281 @@
|
||||
/*
|
||||
* This file is part of the MapleSolaxiaV2 Maple Story Server
|
||||
*
|
||||
* Copyright (C) 2017 RonanLana
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package net.server.audit;
|
||||
|
||||
import java.text.DateFormat;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
import java.util.TimeZone;
|
||||
import java.util.concurrent.ScheduledFuture;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import server.TimerManager;
|
||||
import tools.FilePrinter;
|
||||
import tools.locks.MonitoredEnums;
|
||||
import constants.ServerConstants;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author RonanLana
|
||||
*
|
||||
* This tool has the main purpose of auditing deadlocks throughout the server and must be used only for debugging. The flag is USE_THREAD_TRACKER.
|
||||
*/
|
||||
public class ThreadTracker {
|
||||
private static ThreadTracker instance = null;
|
||||
private final Lock ttLock = new ReentrantLock(true);
|
||||
|
||||
private final Map<Long, List<MonitoredEnums>> threadTracker = new HashMap<>();
|
||||
private final Map<Long, Integer> threadUpdate = new HashMap<>();
|
||||
private final Map<Long, Thread> threads = new HashMap<>();
|
||||
|
||||
private final Map<Long, AtomicInteger> lockCount = new HashMap<>();
|
||||
private final Map<Long, MonitoredEnums> lockIds = new HashMap<>();
|
||||
private final Map<Long, Long> lockThreads = new HashMap<>();
|
||||
private final Map<Long, Byte> lockUpdate = new HashMap<>();
|
||||
|
||||
private final Map<MonitoredEnums, Map<Long, Integer>> locks = new HashMap<>();
|
||||
ScheduledFuture<?> threadTrackerSchedule;
|
||||
|
||||
private String printThreadTrackerState(String dateFormat) {
|
||||
|
||||
Map<MonitoredEnums, List<Integer>> lockValues = new HashMap<>();
|
||||
Set<Long> executingThreads = new HashSet<>();
|
||||
|
||||
for(Map.Entry<Long, AtomicInteger> lock : lockCount.entrySet()) {
|
||||
if(lock.getValue().get() != 0) {
|
||||
executingThreads.add(lockThreads.get(lock.getKey()));
|
||||
|
||||
MonitoredEnums lockId = lockIds.get(lock.getKey());
|
||||
List<Integer> list = lockValues.get(lockId);
|
||||
|
||||
if(list == null) {
|
||||
list = new ArrayList<>();
|
||||
lockValues.put(lockId, list);
|
||||
}
|
||||
|
||||
list.add(lock.getValue().get());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
String s = "----------------------------\r\n" + dateFormat + "\r\n ";
|
||||
s += "Lock-thread usage count:";
|
||||
for(Map.Entry<MonitoredEnums, List<Integer>> lock : lockValues.entrySet()) {
|
||||
s += ("\r\n " + lock.getKey().name() + ": ");
|
||||
|
||||
for(Integer i : lock.getValue()) {
|
||||
s += (i + " ");
|
||||
}
|
||||
}
|
||||
s += "\r\n\r\nThread opened lock path:";
|
||||
|
||||
for(Long tid : executingThreads) {
|
||||
s += "\r\n";
|
||||
for(MonitoredEnums lockid : threadTracker.get(tid)) {
|
||||
s += (lockid.name() + " ");
|
||||
}
|
||||
s += "|";
|
||||
}
|
||||
|
||||
s += "\r\n\r\n";
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
private static String printThreadLog(List<MonitoredEnums> stillLockedPath, String dateFormat) {
|
||||
String s = "----------------------------\r\n" + dateFormat + "\r\n ";
|
||||
for(MonitoredEnums lock : stillLockedPath) {
|
||||
s += (lock.name() + " ");
|
||||
}
|
||||
s += "\r\n\r\n";
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
private static String printThreadStack(StackTraceElement[] list, String dateFormat) {
|
||||
String s = "----------------------------\r\n" + dateFormat + "\r\n";
|
||||
for(int i = 0; i < list.length; i++) {
|
||||
s += (" " + list[i].toString() + "\r\n");
|
||||
}
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
public void accessThreadTracker(boolean update, boolean lock, MonitoredEnums lockId, long lockOid) {
|
||||
ttLock.lock();
|
||||
try {
|
||||
if(update) {
|
||||
if(!lock) { // update tracker
|
||||
List<Long> toRemove = new ArrayList<>();
|
||||
|
||||
for(Long l : threadUpdate.keySet()) {
|
||||
int next = threadUpdate.get(l) + 1;
|
||||
if(next == 4) {
|
||||
List<MonitoredEnums> tt = threadTracker.get(l);
|
||||
|
||||
if(tt.isEmpty()) {
|
||||
toRemove.add(l);
|
||||
} else {
|
||||
DateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy HH:mm:ss");
|
||||
dateFormat.setTimeZone(TimeZone.getTimeZone(ServerConstants.TIMEZONE));
|
||||
String df = dateFormat.format(new Date());
|
||||
|
||||
FilePrinter.print(FilePrinter.DEADLOCK_LOCKS, printThreadLog(tt, df));
|
||||
FilePrinter.print(FilePrinter.DEADLOCK_STACK, printThreadStack(threads.get(l).getStackTrace(), df));
|
||||
}
|
||||
}
|
||||
|
||||
threadUpdate.put(l, next);
|
||||
}
|
||||
|
||||
for(Long l : toRemove) {
|
||||
threadTracker.remove(l);
|
||||
threadUpdate.remove(l);
|
||||
threads.remove(l);
|
||||
|
||||
for(Map<Long, Integer> threadLock : locks.values()) {
|
||||
threadLock.remove(l);
|
||||
}
|
||||
}
|
||||
|
||||
toRemove.clear();
|
||||
|
||||
for(Entry<Long, Byte> it : lockUpdate.entrySet()) {
|
||||
byte val = (byte)(it.getValue() + 1);
|
||||
|
||||
if(val < 60) { // free the structure after 60 silent updates
|
||||
lockUpdate.put(it.getKey(), val);
|
||||
} else {
|
||||
toRemove.add(it.getKey());
|
||||
}
|
||||
}
|
||||
|
||||
for(Long l : toRemove) {
|
||||
lockCount.remove(l);
|
||||
lockIds.remove(l);
|
||||
lockThreads.remove(l);
|
||||
lockUpdate.remove(l);
|
||||
}
|
||||
} else { // print status
|
||||
DateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy HH:mm:ss");
|
||||
dateFormat.setTimeZone(TimeZone.getTimeZone(ServerConstants.TIMEZONE));
|
||||
|
||||
FilePrinter.printError(FilePrinter.DEADLOCK_STATE, printThreadTrackerState(dateFormat.format(new Date())));
|
||||
//FilePrinter.printError(FilePrinter.DEADLOCK_STATE, "[" + dateFormat.format(new Date()) + "] Presenting current lock path for lockid " + lockId.name() + ".\r\n" + printLockStatus(lockId) + "\r\n-------------------------------\r\n");
|
||||
}
|
||||
} else {
|
||||
long tid = Thread.currentThread().getId();
|
||||
|
||||
if(lock) {
|
||||
AtomicInteger c = lockCount.get(lockOid);
|
||||
if(c == null) {
|
||||
c = new AtomicInteger(0);
|
||||
lockCount.put(lockOid, c);
|
||||
lockIds.put(lockOid, lockId);
|
||||
lockThreads.put(lockOid, tid);
|
||||
lockUpdate.put(lockOid, (byte) 0);
|
||||
}
|
||||
c.incrementAndGet();
|
||||
|
||||
List<MonitoredEnums> list = threadTracker.get(tid);
|
||||
if(list == null) {
|
||||
list = new ArrayList<>(20);
|
||||
threadTracker.put(tid, list);
|
||||
threadUpdate.put(tid, 0);
|
||||
threads.put(tid, Thread.currentThread());
|
||||
}
|
||||
list.add(lockId);
|
||||
|
||||
Map<Long, Integer> threadLock = locks.get(lockId);
|
||||
if(threadLock == null) {
|
||||
threadLock = new HashMap<>(5);
|
||||
locks.put(lockId, threadLock);
|
||||
}
|
||||
|
||||
Integer lc = threadLock.get(tid);
|
||||
if(lc != null) {
|
||||
threadLock.put(tid, lc + 1);
|
||||
} else {
|
||||
threadLock.put(tid, 1);
|
||||
}
|
||||
}
|
||||
else {
|
||||
AtomicInteger c = lockCount.get(lockOid);
|
||||
c.decrementAndGet();
|
||||
lockUpdate.put(lockOid, (byte) 0);
|
||||
|
||||
List<MonitoredEnums> list = threadTracker.get(tid);
|
||||
for(int i = list.size() - 1; i >= 0; i--) {
|
||||
if(lockId.getValue() == list.get(i).getValue()) {
|
||||
list.remove(i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Map<Long, Integer> threadLock = locks.get(lockId);
|
||||
threadLock.put(tid, threadLock.get(tid) - 1);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
ttLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private String printLockStatus(MonitoredEnums lockId) {
|
||||
String s = "";
|
||||
|
||||
for(Long threadid : locks.get(lockId).keySet()) {
|
||||
for(MonitoredEnums lockid : threadTracker.get(threadid)) {
|
||||
s += (" " + lockid.name());
|
||||
}
|
||||
|
||||
s += " |\r\n";
|
||||
}
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
public void registerThreadTrackerTask() {
|
||||
threadTrackerSchedule = TimerManager.getInstance().register(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
accessThreadTracker(true, false, MonitoredEnums.UNDEFINED, -1);
|
||||
}
|
||||
}, 10000, 10000);
|
||||
}
|
||||
|
||||
public void cancelThreadTrackerTask() {
|
||||
threadTrackerSchedule.cancel(false);
|
||||
}
|
||||
|
||||
public static ThreadTracker getInstance() {
|
||||
if (instance == null) {
|
||||
instance = new ThreadTracker();
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user