Question

在大数据量数据集做操作时，比如一个数据集有70万个面对象，使用了组件的方法对字段进行了读取名称另存为List<string>的操作，没有增删字段信息。然后再次进行了遍历查询等其他操作，查询使用了多线程，即将这个大数据集放到了多个线程中同时查询。在这个等待查询的过程中，内存占用不断增高，查看Java虚拟机情况看到的是FieldInfo类占用了最大的内存，然后随着运行，内存会逐渐被耗尽，最终程序崩溃，这个是怎么处理？本机内存32g，ddr4.附上崩溃日志截图和虚拟机信息。

Answer 1

您好，请您将多进程查询的代码贴在下面。

Comments

Callable.Callable calltask;
    //局部线程池
    ExecutorService executorService = Executors.newFixedThreadPool(computerCoreCount);
    //拆分任务
    List<String> stringList=new ArrayList<>(stringAndObjectMap.keySet());

    //保存结果
    List<Future<Map<String, Map<String, List<Object>>>>> futures = new ArrayList<>();

    List<List<String>> lists = averageAssign(stringList, processCoreCount);
                                for (List<String> list : lists) {
        if (list.size() < 1) {
            continue;
        }

        calltask=new Callable.Callable(list,fieldNames,stringAndObjectMap,basicInfo);
        futures.add(executorService.submit(calltask));
    }

    public class Callable implements Callable<Map<String, Map<String, List<Object>>>> {
        private Map<String,List<E2PObject>> stringAndObjectMap;
        private List<String> stringList;
        private List<String> fieldNames;
        private basicInfo basicInfo;

        public Callable(List<String> stringList, List<String> fieldNames, Map<String,List<E2PObject>> stringAndObjectMap,basicInfo basicInfo) {
            this.stringList = stringList;
            this.fieldNames = fieldNames;
            this.stringAndObjectMap = stringAndObjectMap;
            this.basicInfo = basicInfo;
        }
        @Override
        public Map<String, Map<String, List<Object>>> call() throws Exception {
            //保存结果
            Map<String, Map<String, List<Object>>> strAndFieldValueMap = new HashMap<>();
            System.out.println("当前线程："+Thread.currentThread().getName() );

            for (String str : stringList) {
                currentCompleteCount.incrementAndGet();
                currentMessage.set("正在整理属性值..."+currentCompleteCount.get()+"/"+currentTotalCount);
                List<E2PObject> e2PObjectList = stringAndObjectMap.get(str);
                for (E2PObject e2PObject : e2PObjectList) {
                    String datasetname = e2PObject.getDatasetname();
                    String BSMValue = e2PObject.getBSM();
                    if (!StrUtilities.isNullOrEmpty(datasetname) && datasource.getDatasets().contains(datasetname) && !StrUtilities.isNullOrEmpty(BSMValue)) {
                        DatasetVector dtV = (DatasetVector) datasource.getDatasets().get(datasetname);
                        if (dtV.getFieldInfos().indexOf(QueryUtility.BSM)>-1) {
                            Recordset rec = dtV.query(QueryUtility.BSM + "='" + BSMValue + "'", CursorType.STATIC);
                            if (rec != null && rec.getRecordCount() > 0) {
                                rec.moveFirst();
                                if (!strAndFieldValueMap.containsKey(str)) {
                                    strAndFieldValueMap.put(str, new LinkedHashMap<String, List<Object>>());
                                    Map<String, List<Object>> fieldAndValues = strAndFieldValueMap.get(str);
                                    for (String fieldName : fieldNames) {
                                        fieldAndValues.put(fieldName, new ArrayList<Object>());
                                        List<Object> objects = fieldAndValues.get(fieldName);
                                        if (rec.getFieldInfos().indexOf(fieldName) > -1) {
                                            Object fieldValue = rec.getFieldValue(fieldName);
                                            if (fieldValue != null && !fieldValue.toString().equals("-999") && !StrUtilities.isNullOrEmpty(fieldValue.toString())) {
                                                if (!objects.contains(fieldValue)) {
                                                    objects.add(fieldValue);
                                                }
                                            }
                                        }

                                    }
                                } else {
                                    Map<String, List<Object>> fieldAndValues = strAndFieldValueMap.get(str);
                                    for (String fieldName : fieldNames) {
                                        if (rec.getFieldInfos().indexOf(fieldName) > -1) {
                                            Object fieldValue = rec.getFieldValue(fieldName);
                                            if (!fieldAndValues.get(fieldName).contains(fieldValue)) {
                                                fieldAndValues.get(fieldName).add(fieldValue);
                                            }
                                        }
                                    }
                                }
                            }
                            if (rec != null) {
                                rec.close();
                                rec.dispose();
                            }
                        }
                    }
                }
            }

            return strAndFieldValueMap;
        }
    }