Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix bug:MultiPagePipeline and DoubleKeyMap concurrent bug

master
edwardsbean 2015-02-13 15:03:13 +08:00
parent 6b9d21fcf3
commit 74962d69b9
2 changed files with 51 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/MultiPagePipeline.java
View File

@ -36,9 +36,15 @@ public class MultiPagePipeline implements Pipeline {
private void handleObject(Iterator<Map.Entry<String, Object>> iterator) {
Map.Entry<String, Object> objectEntry = iterator.next();
Object o = objectEntry.getValue();
//需要拼凑
if (o instanceof MultiPageModel) {
MultiPageModel multiPageModel = (MultiPageModel) o;
//这次处理的部分，设置为完成
pageMap.put(multiPageModel.getPageKey(), multiPageModel.getPage(), Boolean.FALSE);
//每个key单独加锁
synchronized (pageMap.get(multiPageModel.getPageKey())) {
pageMap.put(multiPageModel.getPageKey(), multiPageModel.getPage(), Boolean.TRUE);
//其他需要拼凑的部分
if (multiPageModel.getOtherPages() != null) {
for (String otherPage : multiPageModel.getOtherPages()) {
Boolean aBoolean = pageMap.get(multiPageModel.getPageKey(), otherPage);
@ -53,6 +59,7 @@ public class MultiPagePipeline implements Pipeline {
if (booleanMap == null) {
return;
}
// /过滤，这次完成的page item中，还未拼凑完整的item，不进入下一个pipeline
for (Map.Entry<String, Boolean> stringBooleanEntry : booleanMap.entrySet()) {
if (!stringBooleanEntry.getValue()) {
iterator.remove();
@ -74,6 +81,7 @@ public class MultiPagePipeline implements Pipeline {
}
}
});
// 合并
MultiPageModel value = entryList.get(0).getValue();
for (int i = 1; i < entryList.size(); i++) {
value = value.combine(entryList.get(i).getValue());
@ -84,3 +92,5 @@ public class MultiPagePipeline implements Pipeline {
}
}
}

5
webmagic-extension/src/main/java/us/codecraft/webmagic/utils/DoubleKeyMap.java
View File

@ -75,8 +75,9 @@ public class DoubleKeyMap<K1, K2, V> extends MultiKeyMapBase {
* @param value
* @return value
*/
public V put(K1 key1, K2 key2, V value) {
public synchronized V put(K1 key1, K2 key2, V value) {
if (map.get(key1) == null) {
//不加锁的话，多个线程有可能都会执行到这里
map.put(key1, this.<K2, V>newMap());
}
return get(key1).put(key2, value);
@ -87,7 +88,7 @@ public class DoubleKeyMap<K1, K2, V> extends MultiKeyMapBase {
* @param key2
* @return value
*/
public V remove(K1 key1, K2 key2) {
public synchronized V remove(K1 key1, K2 key2) {
if (get(key1) == null) {
return null;
}