[Wisp] use global coroutine list instead of the coroutine list in eac…

…h thread Summary: Currently, we need to add a lock when we steal a coroutine. If the lock can't be got, the stealing failed. Use global coroutine list to avoid the moving in coroutine list. When we steal the coroutine, we only set the thread in the coroutine. With the change, the success rate has been greatly increased. Test Plan: all wisp cases Reviewed-by: lei.yul, zhengxiaolinX Issue: dragonwell-project/dragonwell8#227
dragonwell-project · Apr 20, 2021 · 3a0dbb1 · 3a0dbb1
1 parent 6db5be9
commit 3a0dbb1
Show file tree

Hide file tree

Showing 13 changed files with 329 additions and 108 deletions.
diff --git a/src/cpu/x86/vm/sharedRuntime_x86_64.cpp b/src/cpu/x86/vm/sharedRuntime_x86_64.cpp
@@ -2364,7 +2364,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
   }
 
   if (EnableCoroutine) {
-    __ movptr(r11, Address(r15_thread, JavaThread::coroutine_list_offset()));
+    __ movptr(r11, Address(r15_thread, JavaThread::thread_coroutine_offset()));
     __ incrementl(Address(r11, Coroutine::native_call_counter_offset()));
   }
 

diff --git a/src/cpu/x86/vm/templateInterpreter_x86_64.cpp b/src/cpu/x86/vm/templateInterpreter_x86_64.cpp
@@ -1210,7 +1210,7 @@ address InterpreterGenerator::generate_native_entry(bool synchronized) {
 #endif
 
   if (EnableCoroutine) {
-    __ movptr(t, Address(r15_thread, JavaThread::coroutine_list_offset()));
+    __ movptr(t, Address(r15_thread, JavaThread::thread_coroutine_offset()));
     __ incrementl(Address(t, Coroutine::native_call_counter_offset()));
   }
 

diff --git a/src/share/vm/gc_implementation/g1/concurrentMark.cpp b/src/share/vm/gc_implementation/g1/concurrentMark.cpp
@@ -2672,7 +2672,7 @@ class CMSATBBufferClosure : public SATBBufferClosure {
   }
 };
 
-class G1RemarkThreadsClosure : public ThreadClosure {
+class G1RemarkThreadsClosure : public ThreadClosure, public CoroutineClosure {
   CMSATBBufferClosure _cm_satb_cl;
   G1CMOopClosure _cm_cl;
   MarkingCodeBlobClosure _code_cl;
@@ -2707,6 +2707,12 @@ class G1RemarkThreadsClosure : public ThreadClosure {
       }
     }
   }
+
+  void do_coroutine(Coroutine* coroutine) {
+    if(coroutine->claim_oops_do(_is_par, _thread_parity)) {
+      coroutine->nmethods_do(&_code_cl);
+    }
+  }
 };
 
 class CMRemarkTask: public AbstractGangTask {
@@ -2726,6 +2732,7 @@ class CMRemarkTask: public AbstractGangTask {
 
         G1RemarkThreadsClosure threads_f(G1CollectedHeap::heap(), task, !_is_serial);
         Threads::threads_do(&threads_f);
+        Coroutine::coroutines_do(&threads_f);
       }
 
       do {

diff --git a/src/share/vm/jfr/leakprofiler/checkpoint/rootResolver.cpp b/src/share/vm/jfr/leakprofiler/checkpoint/rootResolver.cpp
@@ -389,6 +389,13 @@ bool ReferenceToThreadRootClosure::do_java_threads_oops(JavaThread* jt) {
 
   ReferenceLocateClosure rcl(_callback, OldObjectRoot::_threads, OldObjectRoot::_global_jni_handle, jt);
   jt->oops_do(&rcl, NULL, NULL);
+  if (EnableCoroutine) {
+    ALL_JAVA_COROUTINES(c) {
+      if (c->thread() == jt) {
+        c->oops_do(&rcl, NULL, NULL);
+      }
+    }
+  }
   return rcl.complete();
 }
 

diff --git a/src/share/vm/prims/jvm.cpp b/src/share/vm/prims/jvm.cpp
@@ -3567,7 +3567,7 @@ JVM_QUICK_ENTRY(jboolean, JVM_IsInSameNative(JNIEnv* env, jobject jthread))
   // the thread is in native status and the native call counter isn't changed during two calls
   // then return true
   if (thr != NULL && thr->thread_state() == _thread_in_native) {
-    Coroutine* coro = thr->coroutine_list();
+    Coroutine* coro = thr->thread_coroutine();
     assert(coro != NULL, "coroutine list");
     if (coro->last_native_call_counter() == coro->native_call_counter()) {
       return JNI_TRUE;

diff --git a/src/share/vm/prims/unsafe.cpp b/src/share/vm/prims/unsafe.cpp
@@ -1365,7 +1365,7 @@ JVM_ENTRY(jlong, CoroutineSupport_getNativeThreadCoroutine(JNIEnv* env, jclass k
   DEBUG_CORO_PRINT("CoroutineSupport_getNativeThreadCoroutine\n");
   assert(EnableCoroutine, "pre-condition");
 
-  Coroutine* list = thread->coroutine_list();
+  Coroutine* list = thread->thread_coroutine();
   assert(list != NULL, "thread isn't initialized for coroutines");
 
   return (jlong)list;
@@ -1383,11 +1383,10 @@ JVM_ENTRY(void, CoroutineSupport_switchToAndTerminate(JNIEnv* env, jclass klass,
   oop old_oop = JNIHandles::resolve(old_coroutine);
   Coroutine* coro = (Coroutine*)java_dyn_CoroutineBase::data(old_oop);
   assert(coro != NULL, "NULL old coroutine in switchToAndTerminate");
-
   java_dyn_CoroutineBase::set_data(old_oop, 0);
 
   CoroutineStack::free_stack(coro->stack(), thread);
-  delete coro;
+  coro->clean_up();
 JVM_END
 
 JVM_ENTRY(void, CoroutineSupport_switchToAndExit(JNIEnv* env, jclass klass, jobject old_coroutine, jobject target_coroutine))
@@ -1414,7 +1413,7 @@ JVM_ENTRY(jlong, CoroutineSupport_createCoroutine(JNIEnv* env, jclass klass, job
   if (coro == NULL) {
     THROW_0(vmSymbols::java_lang_OutOfMemoryError());
   }
-  coro->insert_into_list(thread->coroutine_list());
+  coro->insert();
   return (jlong)coro;
 JVM_END
 
@@ -1429,7 +1428,7 @@ JVM_ENTRY(jboolean, CoroutineSupport_testDisposableAndTryReleaseStack(JNIEnv* en
   jboolean is_disposable = coro->is_disposable();
   if (is_disposable) {
     CoroutineStack::free_stack(coro->stack(), thread);
-    delete coro;
+    coro->clean_up();
   }
   return is_disposable;
 JVM_END
@@ -1449,18 +1448,18 @@ JVM_ENTRY(void, CoroutineSupport_setWispBooted(JNIEnv* env, jclass klass))
   WispThread::set_wisp_booted(thread);
 JVM_END
 
-JVM_ENTRY (jobject, CoroutineSupport_getNextCoroutine(JNIEnv* env, jclass klass, jlong coroPtr))
+JVM_ENTRY(void, CoroutineSupport_markAsCarrier(JNIEnv* env, jclass klass))
+  DEBUG_CORO_PRINT("CoroutineSupport_markAsCarrier\n");
   assert(EnableCoroutine, "pre-condition");
-  Coroutine* coro = (Coroutine*)coroPtr;
-  assert (coro->next()->coroutine() != NULL, "coroutine oop can't be null");
-  return JNIHandles::make_local(env, coro->next()->coroutine());
+  //Coroutine::mark_as_carrier(thread);
 JVM_END
 
-JVM_ENTRY (void, CoroutineSupport_moveCoroutine(JNIEnv* env, jclass klass, jlong coroPtr, jlong targetPtr))
+
+JVM_ENTRY (jobject, CoroutineSupport_getNextCoroutine(JNIEnv* env, jclass klass, jlong coroPtr))
   assert(EnableCoroutine, "pre-condition");
   Coroutine* coro = (Coroutine*)coroPtr;
-  Coroutine* target = (Coroutine*)targetPtr;
-  Coroutine::move(coro, target);
+  Coroutine* next = coro->next_coroutine(coro->thread());
+  return JNIHandles::make_local(env, next->coroutine());
 JVM_END
 
 JVM_ENTRY (void, CoroutineSupport_markThreadCoroutine(JNIEnv* env, jclass klass, jlong coroPtr, jobject coroObj))
@@ -1486,8 +1485,6 @@ JVM_ENTRY(jboolean, CoroutineSupport_stealCoroutine(JNIEnv* env, jclass klass, j
   }
   assert(coro->thread() != thread, "steal from self");
   assert(coro->state() != Coroutine::_current, "running");
-  coro->remove_from_list(coro->thread()->coroutine_list());
-  coro->insert_into_list(thread->coroutine_list());
   // change thread logic
   if (coro->last_handle_mark() != NULL) {
     coro->last_handle_mark()->change_thread_for_wisp(thread);
@@ -1873,9 +1870,9 @@ JNINativeMethod coroutine_support_methods[] = {
                                   CC"(J)Z",           FN_PTR(CoroutineSupport_testDisposableAndTryReleaseStack)},
     {CC"cleanupCoroutine",        CC"()"COBA,         FN_PTR(CoroutineSupport_cleanupCoroutine)},
     {CC"setWispBooted",           CC"()V",            FN_PTR(CoroutineSupport_setWispBooted)},
+    {CC"markAsCarrier",           CC"()V",            FN_PTR(CoroutineSupport_markAsCarrier)},
     {CC"stealCoroutine",          CC"(J)Z",           FN_PTR(CoroutineSupport_stealCoroutine)},
     {CC"getNextCoroutine",        CC"(J)"COR,         FN_PTR(CoroutineSupport_getNextCoroutine)},
-    {CC"moveCoroutine",           CC"(JJ)V",          FN_PTR(CoroutineSupport_moveCoroutine)},
     {CC"markThreadCoroutine",     CC"(J"COBA")V",     FN_PTR(CoroutineSupport_markThreadCoroutine)},
     {CC"shouldThrowException0", CC"(J)Z",           FN_PTR(CoroutineSupport_shouldThrowException0)},
     {CC"getCoroutineStack",       CC"(J)["STE,        FN_PTR(CoroutineSupport_getCoroutineStack)},

diff --git a/src/share/vm/runtime/arguments.cpp b/src/share/vm/runtime/arguments.cpp
@@ -4146,6 +4146,12 @@ jint Arguments::parse(const JavaVMInitArgs* args) {
   }
 #endif
 
+  if (EnableCoroutine && UseParallelGC) {
+    warning("ParallelGC is not supported with Wisp"
+            "; ignoring EnableCoroutine flag." );
+    FLAG_SET_DEFAULT(EnableCoroutine, false);
+    FLAG_SET_DEFAULT(UseWispMonitor, false);
+  }
   // Set object alignment values.
   set_object_alignment();
 

diff --git a/src/share/vm/runtime/coroutine.cpp b/src/share/vm/runtime/coroutine.cpp
@@ -73,6 +73,10 @@ void coroutine_start(Coroutine* coroutine, jobject coroutineObj) {
 }
 #endif
 
+Coroutine* Coroutine::_coroutineHead = NULL;
+Mutex* Coroutine::_coroutine_list_lock = NULL;
+bool Coroutine::_need_clean_up = false;
+
 void Coroutine::run(jobject coroutine) {
 
   // do not call JavaThread::current() here!
@@ -127,6 +131,9 @@ Coroutine* Coroutine::create_thread_coroutine(JavaThread* thread, CoroutineStack
   coro->_clinit_call_counter = 0;
   coro->_wisp_post_steal_resource_area = NULL;
   coro->_is_yielding  = false;
+  coro->_has_coroutine = false;
+  // This initial value ==> never claimed.
+  coro->_oops_do_parity = 0;
   thread->set_current_coroutine(coro);
   return coro;
 }
@@ -181,15 +188,26 @@ Coroutine* Coroutine::create_coroutine(JavaThread* thread, CoroutineStack* stack
   coro->_clinit_call_counter = 0;
   coro->_wisp_post_steal_resource_area = new (mtWisp) WispResourceArea(coro, 32);
   coro->_is_yielding  = false;
+  coro->_has_coroutine = false;
+  // This initial value ==> never claimed.
+  coro->_oops_do_parity = 0;
+  thread->thread_coroutine()->_has_coroutine = true;
   return coro;
 }
 
 Coroutine::~Coroutine() {
-  remove_from_list(_thread->coroutine_list());
+  assert(_state == Coroutine::_dead, "dead Coroutine");
+  assert(_wisp_thread == NULL, "cleanup");
+  assert(_wisp_post_steal_resource_area == NULL, "cleanup");
+}
+
+void Coroutine::clean_up() {
   if (_wisp_thread != NULL) {
     delete _wisp_thread;
+    _wisp_thread = NULL;
   }
   delete _wisp_post_steal_resource_area;
+  _wisp_post_steal_resource_area = NULL;
   if (!_is_thread_coroutine && _state != Coroutine::_created) {
     assert(_resource_area != NULL, "_resource_area is NULL");
     assert(_handle_area != NULL, "_handle_area is NULL");
@@ -200,6 +218,16 @@ Coroutine::~Coroutine() {
     JNIHandleBlock::release_block(active_handles(), _thread);
     //allocated from JavaCalls::call_virtual during coroutine's first running
   }
+  _thread = NULL;
+  _wisp_task = NULL;
+  _wisp_engine  = NULL;
+  _wisp_task    = NULL;
+  _coroutine = NULL;
+  _state = Coroutine::_dead;
+  _metadata_handles = NULL;
+  _resource_area = NULL;
+  _handle_area = NULL;
+  _need_clean_up = true;
 }
 
 void Coroutine::frames_do(FrameClosure* fc) {
@@ -285,6 +313,88 @@ bool Coroutine::in_critical(JavaThread* thread) {
   return true;
 }
 
+// In Wisp2, the coroutine can be stolen from other carrier.
+// For all the carriers, we mark it as true
+void Coroutine::mark_as_carrier(JavaThread* thread) {
+  thread->thread_coroutine()->_has_coroutine = true;
+}
+
+void Coroutine::coroutines_do(CoroutineClosure* cc) {
+  ALL_JAVA_COROUTINES(c) {
+    cc->do_coroutine(c);
+  }
+}
+
+static void frame_verify(frame* f, const RegisterMap *map) { f->verify(map); }
+
+
+void Coroutine::verify() {
+  if (EnableCoroutine) {
+    oops_do(&VerifyOopClosure::verify_oop, NULL, NULL);
+    frames_do(frame_verify);
+  }
+}
+
+// For global coroutine list, use the thread coroutine of main thread as header
+void Coroutine::init_global_coroutine_list(JavaThread* thread) {
+  thread->thread_coroutine()->insert_into_list(_coroutineHead);
+  _coroutine_list_lock = new Mutex(Mutex::native, "coroutine list lock", false);
+}
+
+void Coroutine::insert() {
+  MutexLockerEx ml(_coroutine_list_lock,  Mutex::_no_safepoint_check_flag);
+  insert_into_list(_coroutineHead);
+}
+
+void Coroutine::delete_exited_coroutines() {
+  assert(SafepointSynchronize::is_at_safepoint(), "should be in safepoint");
+  if (_need_clean_up == false) {
+    return;
+  }
+  _coroutine_list_lock->lock_without_safepoint_check();
+  Coroutine* node = _coroutineHead ? _coroutineHead->next() : NULL;
+  Coroutine* next_node = NULL;
+  for ( ; node != NULL && node != _coroutineHead; ) {
+    next_node = node->next();
+    if (node->_state == _dead && node->_thread == NULL) {
+      node->remove_from_list(_coroutineHead);
+      delete node;
+    }
+    node = next_node;
+  }
+  _coroutine_list_lock->unlock();
+  _need_clean_up = false;
+}
+
+Coroutine* Coroutine::next_coroutine(JavaThread* thread) {
+    Coroutine* node = NULL;
+    int loop = 0;
+    if (_is_thread_coroutine) {
+        if (!_has_coroutine) {
+            return thread->thread_coroutine();
+        }
+        node = _coroutineHead;
+    } else {
+        node = next();
+    }
+    while(node) {
+        if (node == this) {
+            return NULL;
+        } else if (node->_thread == thread) {
+            return node;
+        } else if (node == _coroutineHead) {
+            loop++;
+            if (loop > 1) {
+                return NULL;
+            }
+            node = node->next();
+        } else {
+            node = node->next();
+        }
+    }
+    return thread->thread_coroutine();
+}
+
 class oops_do_Closure: public FrameClosure {
 private:
   OopClosure* _f;
@@ -295,6 +405,25 @@ class oops_do_Closure: public FrameClosure {
   void frames_do(frame* fr, RegisterMap* map) { fr->oops_do(_f, _cld_f, _cf, map); }
 };
 
+// GC support
+bool Coroutine::claim_oops_do_par_case(int strong_roots_parity) {
+  jint coroutine_parity = _oops_do_parity;
+  if (coroutine_parity != strong_roots_parity) {
+    jint res = Atomic::cmpxchg(strong_roots_parity, &_oops_do_parity, coroutine_parity);
+    if (res == coroutine_parity) {
+      return true;
+    } else {
+      guarantee(res == strong_roots_parity, "Or else what?");
+      assert(SharedHeap::heap()->workers()->active_workers() > 0,
+            "Should only fail when parallel.");
+        return false;
+    }
+  }
+  assert(SharedHeap::heap()->workers()->active_workers() > 0,
+        "Should only fail when parallel.");
+  return false;
+}
+
 void Coroutine::oops_do(OopClosure* f, CLDClosure* cld_f, CodeBlobClosure* cf) {
   oops_do_Closure fc(f, cld_f, cf);
   frames_do(&fc);
@@ -1193,7 +1322,7 @@ bool WispPostStealHandleUpdateMark::check(JavaThread *current, bool sp) {
   assert(current == JavaThread::current(), "must be");
   Coroutine *coroutine = current->current_coroutine();
   if (!EnableCoroutine ||
-      coroutine == current->coroutine_list() ||
+      coroutine == current->thread_coroutine() ||
       // if we won't steal it, then no need to allocate handles.
       coroutine->enable_steal_count() != (sp ? current->java_call_counter()+1 : current->java_call_counter())) {
     _success = false;