IndexDoc和RemoveDoc异步调用会导致goroutine锁死

[victorldb]

1. pprof

1 @ 0x43114f 0x441589 0x44155f 0x4412fd 0x466ec9 0x467d3d 0x84b2df 0x9b61ad 0x45de41
#	0x4412fc	sync.runtime_SemacquireMutex+0x3c				~/go/src/runtime/sema.go:71
#	0x466ec8	sync.(*Mutex).Lock+0x108					~/go/src/sync/mutex.go:134
#	0x467d3c	sync.(*RWMutex).Lock+0x2c					~/go/src/sync/rwmutex.go:93
#	0x84b2de	github.com/go-ego/riot/core.(*Indexer).AddDocToCache+0x1ae	~/code/go/src/github.com/go-ego/riot/core/indexer.go:146
#	0x9b61ac	github.com/go-ego/riot.(*Engine).indexerAddDoc+0xac		~/code/go/src/github.com/go-ego/riot/indexer_worker.go:49

1 @ 0x43114f 0x441589 0x44155f 0x4412fd 0x466ec9 0x467d3d 0x84c3f5 0x9b62cb 0x45de41
#	0x4412fc	sync.runtime_SemacquireMutex+0x3c				~/go/src/runtime/sema.go:71
#	0x466ec8	sync.(*Mutex).Lock+0x108					~/go/src/sync/mutex.go:134
#	0x467d3c	sync.(*RWMutex).Lock+0x2c					~/go/src/sync/rwmutex.go:93
#	0x84c3f4	github.com/go-ego/riot/core.(*Indexer).RemoveDocToCache+0x64	~/code/go/src/github.com/go-ego/riot/core/indexer.go:267
#	0x9b62ca	github.com/go-ego/riot.(*Engine).indexerRemoveDoc+0xba		~/code/go/src/github.com/go-ego/riot/indexer_worker.go:65

2.问题代码

func (indexer *Indexer) AddDocToCache(doc *types.DocIndex, forceUpdate bool) {
	if indexer.initialized == false {
		log.Fatal("The Indexer has not been initialized.")
	}

	indexer.addCacheLock.Lock()
	if doc != nil {
		indexer.addCacheLock.addCache[indexer.addCacheLock.addCachePointer] = doc
		indexer.addCacheLock.addCachePointer++
	}

	docSize := indexer.addCacheLock.addCachePointer >= indexer.initOptions.DocCacheSize
	if docSize || forceUpdate {
		indexer.tableLock.Lock()

		position := 0
		for i := 0; i < indexer.addCacheLock.addCachePointer; i++ {
			docIndex := indexer.addCacheLock.addCache[i]

			docState, ok := indexer.tableLock.docsState[docIndex.DocId]
			if ok && docState <= 1 {
				// ok && docState == 0 表示存在于索引中，需先删除再添加
				// ok && docState == 1 表示不一定存在于索引中，等待删除，需先删除再添加
				if position != i {
					indexer.addCacheLock.addCache[position], indexer.addCacheLock.addCache[i] =
						indexer.addCacheLock.addCache[i], indexer.addCacheLock.addCache[position]
				}

				if docState == 0 {
					// delete docs
					indexer.removeCacheLock.Lock()
					indexer.removeCacheLock.removeCache[indexer.removeCacheLock.removeCachePointer] =
						docIndex.DocId
					indexer.removeCacheLock.removeCachePointer++
					indexer.removeCacheLock.Unlock()

					indexer.tableLock.docsState[docIndex.DocId] = 1
					indexer.numDocs--
				}
				position++
			} else if !ok {
				indexer.tableLock.docsState[docIndex.DocId] = 2
			}
		}

		indexer.tableLock.Unlock()
		if indexer.RemoveDocToCache("0", forceUpdate) {
			// 只有当存在于索引表中的文档已被删除，其才可以重新加入到索引表中
			position = 0
		}

		addCachedDocs := indexer.addCacheLock.addCache[position:indexer.addCacheLock.addCachePointer]
		indexer.addCacheLock.addCachePointer = position

		indexer.addCacheLock.Unlock()
		sort.Sort(addCachedDocs)
		indexer.AddDocs(&addCachedDocs)
	} else {
		indexer.addCacheLock.Unlock()
	}
}

3.分析
IndexDoc中的indexer.removeCacheLock.Lock()需要和RemoveDoc中的indexer.removeCacheLock.Lock()一样放在indexer.tableLock.Lock()之前，要不然会导致goroutine锁死
4.修复后代码

func (indexer *Indexer) AddDocToCache(doc *types.DocIndex, forceUpdate bool) {
	if indexer.initialized == false {
		log.Fatal("The Indexer has not been initialized.")
	}

	indexer.addCacheLock.Lock()
	if doc != nil {
		indexer.addCacheLock.addCache[indexer.addCacheLock.addCachePointer] = doc
		indexer.addCacheLock.addCachePointer++
	}

	docSize := indexer.addCacheLock.addCachePointer >= indexer.initOptions.DocCacheSize
	if docSize || forceUpdate {
		indexer.removeCacheLock.Lock()
		indexer.tableLock.Lock()

		position := 0
		for i := 0; i < indexer.addCacheLock.addCachePointer; i++ {
			docIndex := indexer.addCacheLock.addCache[i]

			docState, ok := indexer.tableLock.docsState[docIndex.DocId]
			if ok && docState <= 1 {
				// ok && docState == 0 表示存在于索引中，需先删除再添加
				// ok && docState == 1 表示不一定存在于索引中，等待删除，需先删除再添加
				if position != i {
					indexer.addCacheLock.addCache[position], indexer.addCacheLock.addCache[i] =
						indexer.addCacheLock.addCache[i], indexer.addCacheLock.addCache[position]
				}

				if docState == 0 {
					// delete docs
					indexer.removeCacheLock.removeCache[indexer.removeCacheLock.removeCachePointer] =
						docIndex.DocId
					indexer.removeCacheLock.removeCachePointer++

					indexer.tableLock.docsState[docIndex.DocId] = 1
					indexer.numDocs--
				}
				position++
			} else if !ok {
				indexer.tableLock.docsState[docIndex.DocId] = 2
			}
		}

		indexer.tableLock.Unlock()
		indexer.removeCacheLock.Unlock()
		if indexer.RemoveDocToCache("0", forceUpdate) {
			// 只有当存在于索引表中的文档已被删除，其才可以重新加入到索引表中
			position = 0
		}

		addCachedDocs := indexer.addCacheLock.addCache[position:indexer.addCacheLock.addCachePointer]
		indexer.addCacheLock.addCachePointer = position

		indexer.addCacheLock.Unlock()
		sort.Sort(addCachedDocs)
		indexer.AddDocs(&addCachedDocs)
	} else {
		indexer.addCacheLock.Unlock()
	}
}

[xrfinbupt]

@victorldb why not pull request