(1)首先构造一个实体类,来记录要更新数量,成功数量,失败数量,在途数量。
public class StateNumCounter implements Serializable {
public long succeedNum;
public long sumbitNum;
public long failedNum;
public long pendingNum;
public long pendingNumSubtr;
public long countNum;
public int commitTimes;
private Long batchId;
private Long cellId;
private Long processingCellId;
public Long getProcessingCellId() {
return processingCellId;
}
public void setProcessingCellId(Long processingCellId) {
this.processingCellId = processingCellId;
}
public Long getBatchId() {
return batchId;
}
public void setBatchId(Long batchId) {
this.batchId = batchId;
}
public Long getCellId() {
return cellId;
}
public void setCellId(Long cellId) {
this.cellId = cellId;
}
public long getSucceedNum() {
return succeedNum;
}
public void setSucceedNum(long succeedNum) {
this.succeedNum = succeedNum;
}
public long getSumbitNum() {
return sumbitNum;
}
public void setSumbitNum(long sumbitNum) {
this.sumbitNum = sumbitNum;
}
public long getFailedNum() {
return failedNum;
}
public void setFailedNum(long failedNum) {
this.failedNum = failedNum;
}
public long getPendingNum() {
return pendingNum;
}
public void setPendingNum(long pendingNum) {
this.pendingNum = pendingNum;
}
public long getPendingNumSubtr() {
return pendingNumSubtr;
}
public void setPendingNumSubtr(long pendingNumSubtr) {
this.pendingNumSubtr = pendingNumSubtr;
}
public long getCountNum() {
return countNum;
}
public void setCountNum(long countNum) {
this.countNum = countNum;
}
public int getCommitTimes() {
return commitTimes;
}
public void setCommitTimes(int commitTimes) {
this.commitTimes = commitTimes;
}
(2)写一个用于将数据批量提交持久化库的业务单元。
public final class MessagePersistenceProcess {
private static ZSmartLogger logger = ZSmartLogger.getLogger(MessagePersistenceProcess.class);
private static volatile MessagePersistenceProcess instance = new MessagePersistenceProcess();
private ArrayBlockingQueue
(3).写一个抽象类,每5分钟扫描一次堆栈。
这里定义一个,ArrayBlockingQueue消息队列,发送的短信都会推送到该队列中,队列中数据达到1000条或者超过5分钟,统一处理一次。
public abstract class AbstractStorageHandlerBaseextends Thread { private static ZLogger logger = ZLoggerFactory.getLogger(AbstractStorageHandlerBase.class, "AbstractStorageHandlerBase"); ArrayBlockingQueue queue; private List tempList = new ArrayList<>(); public AbstractStorageHandlerBase(ArrayBlockingQueue queue) { this.queue = queue; } @Override public void run() { long begin = 0L; for (;;) { try { // 队列中有数据且tempList的容量未达到1000,可继续往tempList中放入队列数据 if (queue.size() > 0 && tempList.size() < 1000) { for (; queue.size() > 0 && tempList.size() < 1000;) { tempList.add(queue.take()); } begin = System.currentTimeMillis(); } if (ValidateUtil.validateNotEmpty(tempList)) { long cur = System.currentTimeMillis(); // 队列数据大于等于1000条 或者 5分钟内没有新消息且队列未满1000 则做批量插入操作 if (tempList.size() >= 1000) { process(tempList); tempList.clear(); } else if (begin != 0L && cur - begin > 300000) { process(tempList); tempList.clear(); } Thread.sleep(1000); } else { Thread.sleep(100); } } catch (Exception e) { logger.error(ErrorValues.SMART_ERROR, e, ""); } } } @Transactional public abstract void process(List tempList); }
(4).BatchHandler继承抽象类AbstractStorageHandlerBase,来首先业务功能需求。
public class BatchHandler extends AbstractStorageHandlerBase{ private static ZSmartLogger logger = ZSmartLogger.getLogger(BatchHandler.class); private Map cellResultMap = new HashMap (); private Map batchResultMap = new HashMap (); public BatchHandler(ArrayBlockingQueue queue) { super(queue); } private IMccContactStatisticsDataServ contactStatisticsDataServ = SpringUtil.getBean(IMccContactStatisticsDataServ.class); @Override public void process(List tempList) { for (Map dict : tempList) { processBatchStatisc(dict); } // 更新batch和batch cell的统计信息 contactStatisticsDataServ.updateBatchAndBatchCellStatistics(batchResultMap, cellResultMap); cleanData(); } private void processBatchStatisc(Map updateInfo) { logger.debug("MarketingEventUpdateProcess processBatchStatisc begin... data = {}", updateInfo); Long cellid = MapUtils.getLong(updateInfo, "CELL_ID"); Long batchId = MapUtils.getLong(updateInfo, "BATCH_ID"); StateNumCounter stateCounter = (StateNumCounter) updateInfo.get("COUNTER"); if (!batchResultMap.containsKey(batchId)) { batchResultMap.put(batchId, new StateNumCounter()); } // 取出该 batch Id对应的计数器 StateNumCounter batchCounter = batchResultMap.get(batchId); batchCounter.setCountNum(batchCounter.getCountNum() + stateCounter.getCountNum()); batchCounter.setSucceedNum(batchCounter.getSucceedNum() + stateCounter.getSucceedNum()); batchCounter.setFailedNum(batchCounter.getFailedNum() + stateCounter.getFailedNum()); batchCounter.setPendingNum(batchCounter.getPendingNum() + stateCounter.getPendingNum()); if (cellid != null) { String cellKey = cellid.toString() + "|" + batchId.toString(); if (!cellResultMap.containsKey(cellKey)) { cellResultMap.put(cellKey, new StateNumCounter()); } // 取出该 cell Id对应的计数器 StateNumCounter cellCounter = cellResultMap.get(cellKey); cellCounter.setCountNum(cellCounter.getCountNum() + stateCounter.getCountNum()); cellCounter.setSucceedNum(cellCounter.getSucceedNum() + stateCounter.getSucceedNum()); cellCounter.setFailedNum(cellCounter.getFailedNum() + stateCounter.getFailedNum()); cellCounter.setPendingNum(cellCounter.getPendingNum() + stateCounter.getPendingNum()); logger.debug("MarketingEventUpdateProcess processBatchStatisc end..."); } } private void cleanData() { logger.debug("MarketingEventUpdateProcess cleanData begin..."); batchResultMap.clear(); cellResultMap.clear(); logger.debug("MarketingEventUpdateProcess cleanData end..."); } }
(6).实际业务场景中,使用MessagePersistenceProcess.getInstance().pushBatchStatis(data);
异步推送数据.
//更新contact状态并且更新batch数量
SpringTransactionUtil.doInNewTransaction(transactionStatus -> {
mccContactMapper.batchInsert(mccFailOverContactList);
mccContactDataServ.batchUpdateFailOver(mccContactList);
Map cellResultMap = new HashMap<>();
for (MccContact mccContact : mccFailOverContactList) {
Long batchId = mccContact.getBatchId();
String key = batchId.toString();
if (!cellResultMap.containsKey(key)) {
cellResultMap.put(key, new StateNumCounter());
}
if (CmsStateDef.CONTACT_STATE_FLD.equals(mccContact.getContactState())) {
StateNumCounter stateCounter = cellResultMap.get(key);
stateCounter.failedNum++;
stateCounter.countNum++;
}
else {
StateNumCounter stateCounter = cellResultMap.get(key);
stateCounter.succeedNum++;
stateCounter.countNum++;
}
}
logger.info("ViberReceiptRunnable updateContactBatchAndCell start, cellResultMap:{}",
JSONObject.toJSONString(cellResultMap));
if (!cellResultMap.isEmpty()) {
for (Map.Entry entry : cellResultMap.entrySet()) {
String key = entry.getKey();
StateNumCounter cellsCounter = entry.getValue();
Long batchId = Long.valueOf(key);
// 异步的,入queue
Map data = new HashMap();
data.put("BATCH_ID", batchId);
data.put("COUNTER", cellsCounter);
MessagePersistenceProcess.getInstance().pushBatchStatis(data);
}
}
logger.info("ViberReceiptRunnable updateContactBatchAndCell ends");
return null;
});
注意:启动系统的时候要启动业务单元。
static {
// 启动持久化单元
MessagePersistenceProcess.getInstance().start();
}
本文章仅仅用于记录学习过程和笔记,如果读者看不懂,勿喷。



