硬件环境:html
广州华为服务器 RH2288V3 java
32G 内存,4核,1T硬盘web
软件环境:apache
JDK1.7 weblogic12 Centos6.5服务器
事情通过:app
2017年10月30号晚上有个同事忽然打电话说新版本发布后系统常常运行几天就平白无故崩溃,让我这边帮忙定位下是否是程序问题。通常这种程序崩溃基本上都是内存泄露,但检查最近更新的代码并无发现有什么问题,保险起见赶忙上服务器down 一个dump文件下来观察一下。ui
拿到最新的dump文件以后用MAT分析,一看果真发现内存居高不下(内存占用达8G)。仔细观察问题的根源居然是jstl!由于公司的项目运行多年,用的技术五花八门其中有很多页面用到JSTL..如下就是Memory Analyzer的截图:this
能够看到ELEvaluator这个类有一个Map对象,里面的对象占用内存达到67108880b(67M)该对象,其直接或间接引用的内存达7152269600b(7G)因此基本能够判定致使内存泄露的元凶就是它了。而后查看该对象的源码发现里面果真有2个静态Map对象lua
并且很神奇的是这2个map对象都只有put和get可是没有删除的方法,也就是说这2个对象只会递增!同时由于这2个对象非public也就是说其它类也没法引用...这只能说是JSTL的一个BUG。而后仔细观察这2个对象使用的地方:spa
发现只要获取不到就会建立一个新的对象,也就是说这2个对象都可以删除。因而解决的办法来了:直接把sCachedExpressionStrings和sCachedExpectedTypes删掉。而后从新生成class并替换对应jar包的class。更新上去运行1个星期以后发现内存基本稳定在3G左右(高峰6G低峰1G)。
后来网上查找一下发现老外早就发现这个问题:
http://www.archivum.info/issues@commons.apache.org/2007-09/00118/(jira)-Commented-(EL-1)-(el)-Memory-Leak-EL-Cache-entries-never-removed.html
只是一直没人给出解决的方法,网上也有人说这个BUG是JSTL1.2的问题,以后的版本已经解决。可是由于项目已经交付,若是临时替换JAR包难保不会引起其它问题因此就先这样解决,之后有时间把JSTL从项目中移除出去。
*由于公司用的是weblogic,weblogic自身有不少jar包其中weblogic.server.merged.jar 把JSTL1.2包含进来,若是是使用weblogic作为服务器则要当心须要修改weblogic.server.merged.jar这里面的JSTL,否则改了也不会生效。由于按java类加载顺序,服务器的jar包加载顺序优先于项目的jar包因此若是用的是weblogic的同窗则要当心这个坑。
最后附上ELEvaluator.java的源码,有兴趣的同窗能够研究下有没有更好的解决方案,有问题能够email我。个人邮箱地址是:hurrican_ok@126.com。由于是用jdgui.exe反编译的你们将就看下
package org.apache.taglibs.standard.lang.jstl; import java.io.Reader; import java.io.StringReader; import java.text.MessageFormat; import java.util.Collections; import java.util.HashMap; import java.util.Map; import org.apache.taglibs.standard.lang.jstl.parser.ELParser; import org.apache.taglibs.standard.lang.jstl.parser.ParseException; import org.apache.taglibs.standard.lang.jstl.parser.Token; import org.apache.taglibs.standard.lang.jstl.parser.TokenMgrError; public class ELEvaluator { static Map sCachedExpressionStrings = Collections.synchronizedMap(new HashMap()); static Map sCachedExpectedTypes = new HashMap(); static Logger sLogger = new Logger(System.out); VariableResolver mResolver; boolean mBypassCache; public ELEvaluator(VariableResolver pResolver) { this.mResolver = pResolver; } public ELEvaluator(VariableResolver pResolver, boolean pBypassCache) { this.mResolver = pResolver; this.mBypassCache = pBypassCache; } public Object evaluate(String pExpressionString, Object pContext, Class pExpectedType, Map functions, String defaultPrefix) throws ELException { return evaluate(pExpressionString, pContext, pExpectedType, functions, defaultPrefix, sLogger); } Object evaluate(String pExpressionString, Object pContext, Class pExpectedType, Map functions, String defaultPrefix, Logger pLogger) throws ELException { if (pExpressionString == null) { throw new ELException(Constants.NULL_EXPRESSION_STRING); } Object parsedValue = parseExpressionString(pExpressionString); if ((parsedValue instanceof String)) { String strValue = (String)parsedValue; return convertStaticValueToExpectedType(strValue, pExpectedType, pLogger); } if ((parsedValue instanceof Expression)) { Object value = ((Expression)parsedValue).evaluate(pContext, this.mResolver, functions, defaultPrefix, pLogger); return convertToExpectedType(value, pExpectedType, pLogger); } if ((parsedValue instanceof ExpressionString)) { String strValue = ((ExpressionString)parsedValue).evaluate(pContext, this.mResolver, functions, defaultPrefix, pLogger); return convertToExpectedType(strValue, pExpectedType, pLogger); } return null; } public Object parseExpressionString(String pExpressionString) throws ELException { if (pExpressionString.length() == 0) { return ""; } Object ret = this.mBypassCache ? null : sCachedExpressionStrings.get(pExpressionString); if (ret == null) { Reader r = new StringReader(pExpressionString); ELParser parser = new ELParser(r); try { ret = parser.ExpressionString(); sCachedExpressionStrings.put(pExpressionString, ret); } catch (ParseException exc) { throw new ELException(formatParseException(pExpressionString, exc)); } catch (TokenMgrError exc) { throw new ELException(exc.getMessage()); } } return ret; } Object convertToExpectedType(Object pValue, Class pExpectedType, Logger pLogger) throws ELException { return Coercions.coerce(pValue, pExpectedType, pLogger); } Object convertStaticValueToExpectedType(String pValue, Class pExpectedType, Logger pLogger) throws ELException { if ((pExpectedType == String.class) || (pExpectedType == Object.class)) { return pValue; } Map valueByString = getOrCreateExpectedTypeMap(pExpectedType); if ((!this.mBypassCache) && (valueByString.containsKey(pValue))) { return valueByString.get(pValue); } Object ret = Coercions.coerce(pValue, pExpectedType, pLogger); valueByString.put(pValue, ret); return ret; } static Map getOrCreateExpectedTypeMap(Class pExpectedType) { synchronized (sCachedExpectedTypes) { Map ret = (Map)sCachedExpectedTypes.get(pExpectedType); if (ret == null) { ret = Collections.synchronizedMap(new HashMap()); sCachedExpectedTypes.put(pExpectedType, ret); } return ret; } } static String formatParseException(String pExpressionString, ParseException pExc) { StringBuffer expectedBuf = new StringBuffer(); int maxSize = 0; boolean printedOne = false; if (pExc.expectedTokenSequences == null) { return pExc.toString(); } for (int i = 0; i < pExc.expectedTokenSequences.length; i++) { if (maxSize < pExc.expectedTokenSequences[i].length) { maxSize = pExc.expectedTokenSequences[i].length; } for (int j = 0; j < pExc.expectedTokenSequences[i].length; j++) { if (printedOne) { expectedBuf.append(", "); } expectedBuf.append(pExc.tokenImage[pExc.expectedTokenSequences[i][j]]); printedOne = true; } } String expected = expectedBuf.toString(); StringBuffer encounteredBuf = new StringBuffer(); Token tok = pExc.currentToken.next; for (int i = 0; i < maxSize; i++) { if (i != 0) { encounteredBuf.append(" "); } if (tok.kind == 0) { encounteredBuf.append(pExc.tokenImage[0]); break; } encounteredBuf.append(addEscapes(tok.image)); tok = tok.next; } String encountered = encounteredBuf.toString(); return MessageFormat.format(Constants.PARSE_EXCEPTION, new Object[] { expected, encountered }); } static String addEscapes(String str) { StringBuffer retval = new StringBuffer(); for (int i = 0; i < str.length(); i++) { switch (str.charAt(i)) { case '\000': break; case '\b': retval.append("\\b"); break; case '\t': retval.append("\\t"); break; case '\n': retval.append("\\n"); break; case '\f': retval.append("\\f"); break; case '\r': retval.append("\\r"); break; case '\001': case '\002': case '\003': case '\004': case '\005': case '\006': case '\007': case '\013': default: char ch; if (((ch = str.charAt(i)) < ' ') || (ch > '~')) { String s = "0000" + Integer.toString(ch, 16); retval.append("\\u" + s.substring(s.length() - 4, s.length())); } else { retval.append(ch); } break; } } return retval.toString(); } public String parseAndRender(String pExpressionString) throws ELException { Object val = parseExpressionString(pExpressionString); if ((val instanceof String)) { return (String)val; } if ((val instanceof Expression)) { return "${" + ((Expression)val).getExpressionString() + "}"; } if ((val instanceof ExpressionString)) { return ((ExpressionString)val).getExpressionString(); } return ""; } }