From 84954858239f3867df537543f937c827c484697f Mon Sep 17 00:00:00 2001 From: Ronen Nachmias Date: Sat, 12 May 2018 15:59:21 +0300 Subject: [PATCH] first commit --- .gitignore | 31 + .travis.yml | 27 + CHANGES.md | 64 ++ CODE_OF_CONDUCT.md | 46 ++ DEVELOPMENT.md | 20 + LICENSE.txt | 202 +++++ README.md | 187 +++++ benchmark/pom.xml | 77 ++ .../main/java/cache/CacheReadBenchmark.java | 65 ++ .../gossip/ProtostuffGossipBenchmark.java | 95 +++ .../transport/ProtostuffMessageBenchmark.java | 77 ++ benchmark/test/pom.xml | 171 ++++ checkstyle-suppression.xml | 10 + checkstyle.xml | 208 +++++ cluster/pom.xml | 26 + .../java/io/scalecube/cluster/Cluster.java | 202 +++++ .../io/scalecube/cluster/ClusterConfig.java | 392 ++++++++++ .../io/scalecube/cluster/ClusterImpl.java | 242 ++++++ .../io/scalecube/cluster/ClusterMath.java | 60 ++ .../java/io/scalecube/cluster/Member.java | 86 +++ .../cluster/fdetector/FailureDetector.java | 30 + .../fdetector/FailureDetectorConfig.java | 11 + .../fdetector/FailureDetectorEvent.java | 33 + .../fdetector/FailureDetectorImpl.java | 388 ++++++++++ .../scalecube/cluster/fdetector/PingData.java | 50 ++ .../io/scalecube/cluster/gossip/Gossip.java | 54 ++ .../cluster/gossip/GossipConfig.java | 11 + .../cluster/gossip/GossipProtocol.java | 39 + .../cluster/gossip/GossipProtocolImpl.java | 309 ++++++++ .../cluster/gossip/GossipRequest.java | 35 + .../scalecube/cluster/gossip/GossipState.java | 49 ++ .../cluster/membership/MemberStatus.java | 21 + .../cluster/membership/MembershipConfig.java | 28 + .../cluster/membership/MembershipEvent.java | 79 ++ .../membership/MembershipProtocol.java | 37 + .../membership/MembershipProtocolImpl.java | 561 ++++++++++++++ .../cluster/membership/MembershipRecord.java | 120 +++ .../cluster/membership/SyncData.java | 49 ++ .../io/scalecube/cluster/ClusterTest.java | 207 +++++ .../fdetector/FailureDetectorTest.java | 472 +++++++++++ .../cluster/gossip/GossipProtocolTest.java | 267 +++++++ .../cluster/gossip/GossipRequestTest.java | 96 +++ .../membership/DummyMembershipProtocol.java | 53 ++ .../membership/MembershipProtocolTest.java | 598 ++++++++++++++ .../membership/MembershipRecordTest.java | 106 +++ eclipse-java-google-style.xml | 730 ++++++++++++++++++ eclipse.importorder | 9 + examples/pom.xml | 24 + .../examples/ClusterJoinExamples.java | 67 ++ .../examples/ClusterMetadataExample.java | 54 ++ .../io/scalecube/examples/GossipExample.java | 39 + .../examples/MembershipEventsExample.java | 60 ++ .../scalecube/examples/MessagingExample.java | 49 ++ pom.xml | 370 +++++++++ src/main/scripts/cd/before-deploy.sh | 62 ++ src/main/scripts/cd/codesigning.asc.enc | Bin 0 -> 9584 bytes src/main/scripts/cd/deploy.sh | 9 + src/main/scripts/cd/secrets.tar.enc | Bin 0 -> 20496 bytes src/main/scripts/ci/before-install.sh | 19 + style.importorder | 10 + testlib/pom.xml | 45 ++ .../java/io/scalecube/testlib/BaseTest.java | 32 + testlib/src/main/resources/log4j2.xml | 23 + transport/pom.xml | 105 +++ .../scalecube/transport/BootstrapFactory.java | 115 +++ .../scalecube/transport/ExceptionHandler.java | 29 + .../java/io/scalecube/transport/Message.java | 271 +++++++ .../io/scalecube/transport/MessageCodec.java | 66 ++ .../transport/MessageDeserializerHandler.java | 18 + .../scalecube/transport/MessageHandler.java | 38 + .../io/scalecube/transport/MessageSchema.java | 230 ++++++ .../transport/MessageSerializerHandler.java | 16 + .../scalecube/transport/NetworkEmulator.java | 223 ++++++ .../transport/NetworkEmulatorException.java | 21 + .../transport/NetworkEmulatorHandler.java | 68 ++ .../transport/NetworkLinkSettings.java | 78 ++ .../transport/RecyclableLinkedBuffer.java | 68 ++ .../io/scalecube/transport/Transport.java | 146 ++++ .../scalecube/transport/TransportConfig.java | 214 +++++ .../io/scalecube/transport/TransportImpl.java | 326 ++++++++ .../transport/NetworkEmulatorTest.java | 44 ++ .../transport/TransportSendOrderTest.java | 238 ++++++ .../io/scalecube/transport/TransportTest.java | 450 +++++++++++ .../transport/TransportTestUtils.java | 57 ++ travis-settings.xml | 26 + utils/pom.xml | 30 + .../cluster/membership/IdGenerator.java | 72 ++ .../java/io/scalecube/transport/Address.java | 94 +++ .../io/scalecube/transport/Addressing.java | 160 ++++ .../java/io/scalecube/utils/AddressTest.java | 33 + .../io/scalecube/utils/IdGeneratorTest.java | 33 + 91 files changed, 10862 insertions(+) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 CHANGES.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 DEVELOPMENT.md create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 benchmark/pom.xml create mode 100644 benchmark/src/main/java/cache/CacheReadBenchmark.java create mode 100644 benchmark/src/main/java/io/scalecube/cluster/gossip/ProtostuffGossipBenchmark.java create mode 100644 benchmark/src/main/java/io/scalecube/transport/ProtostuffMessageBenchmark.java create mode 100644 benchmark/test/pom.xml create mode 100644 checkstyle-suppression.xml create mode 100644 checkstyle.xml create mode 100644 cluster/pom.xml create mode 100644 cluster/src/main/java/io/scalecube/cluster/Cluster.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/ClusterConfig.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/ClusterImpl.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/ClusterMath.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/Member.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/fdetector/FailureDetector.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/fdetector/FailureDetectorConfig.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/fdetector/FailureDetectorEvent.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/fdetector/FailureDetectorImpl.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/fdetector/PingData.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/gossip/Gossip.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/gossip/GossipConfig.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/gossip/GossipProtocol.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/gossip/GossipProtocolImpl.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/gossip/GossipRequest.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/gossip/GossipState.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/membership/MemberStatus.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/membership/MembershipConfig.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/membership/MembershipEvent.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/membership/MembershipProtocol.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/membership/MembershipProtocolImpl.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/membership/MembershipRecord.java create mode 100644 cluster/src/main/java/io/scalecube/cluster/membership/SyncData.java create mode 100644 cluster/src/test/java/io/scalecube/cluster/ClusterTest.java create mode 100644 cluster/src/test/java/io/scalecube/cluster/fdetector/FailureDetectorTest.java create mode 100644 cluster/src/test/java/io/scalecube/cluster/gossip/GossipProtocolTest.java create mode 100644 cluster/src/test/java/io/scalecube/cluster/gossip/GossipRequestTest.java create mode 100644 cluster/src/test/java/io/scalecube/cluster/membership/DummyMembershipProtocol.java create mode 100644 cluster/src/test/java/io/scalecube/cluster/membership/MembershipProtocolTest.java create mode 100644 cluster/src/test/java/io/scalecube/cluster/membership/MembershipRecordTest.java create mode 100644 eclipse-java-google-style.xml create mode 100644 eclipse.importorder create mode 100644 examples/pom.xml create mode 100644 examples/src/main/java/io/scalecube/examples/ClusterJoinExamples.java create mode 100644 examples/src/main/java/io/scalecube/examples/ClusterMetadataExample.java create mode 100644 examples/src/main/java/io/scalecube/examples/GossipExample.java create mode 100644 examples/src/main/java/io/scalecube/examples/MembershipEventsExample.java create mode 100644 examples/src/main/java/io/scalecube/examples/MessagingExample.java create mode 100644 pom.xml create mode 100644 src/main/scripts/cd/before-deploy.sh create mode 100644 src/main/scripts/cd/codesigning.asc.enc create mode 100644 src/main/scripts/cd/deploy.sh create mode 100644 src/main/scripts/cd/secrets.tar.enc create mode 100644 src/main/scripts/ci/before-install.sh create mode 100644 style.importorder create mode 100644 testlib/pom.xml create mode 100644 testlib/src/main/java/io/scalecube/testlib/BaseTest.java create mode 100644 testlib/src/main/resources/log4j2.xml create mode 100644 transport/pom.xml create mode 100644 transport/src/main/java/io/scalecube/transport/BootstrapFactory.java create mode 100644 transport/src/main/java/io/scalecube/transport/ExceptionHandler.java create mode 100644 transport/src/main/java/io/scalecube/transport/Message.java create mode 100644 transport/src/main/java/io/scalecube/transport/MessageCodec.java create mode 100644 transport/src/main/java/io/scalecube/transport/MessageDeserializerHandler.java create mode 100644 transport/src/main/java/io/scalecube/transport/MessageHandler.java create mode 100644 transport/src/main/java/io/scalecube/transport/MessageSchema.java create mode 100644 transport/src/main/java/io/scalecube/transport/MessageSerializerHandler.java create mode 100644 transport/src/main/java/io/scalecube/transport/NetworkEmulator.java create mode 100644 transport/src/main/java/io/scalecube/transport/NetworkEmulatorException.java create mode 100644 transport/src/main/java/io/scalecube/transport/NetworkEmulatorHandler.java create mode 100644 transport/src/main/java/io/scalecube/transport/NetworkLinkSettings.java create mode 100644 transport/src/main/java/io/scalecube/transport/RecyclableLinkedBuffer.java create mode 100644 transport/src/main/java/io/scalecube/transport/Transport.java create mode 100644 transport/src/main/java/io/scalecube/transport/TransportConfig.java create mode 100644 transport/src/main/java/io/scalecube/transport/TransportImpl.java create mode 100644 transport/src/test/java/io/scalecube/transport/NetworkEmulatorTest.java create mode 100644 transport/src/test/java/io/scalecube/transport/TransportSendOrderTest.java create mode 100644 transport/src/test/java/io/scalecube/transport/TransportTest.java create mode 100644 transport/src/test/java/io/scalecube/transport/TransportTestUtils.java create mode 100644 travis-settings.xml create mode 100644 utils/pom.xml create mode 100644 utils/src/main/java/io/scalecube/cluster/membership/IdGenerator.java create mode 100644 utils/src/main/java/io/scalecube/transport/Address.java create mode 100644 utils/src/main/java/io/scalecube/transport/Addressing.java create mode 100644 utils/src/test/java/io/scalecube/utils/AddressTest.java create mode 100644 utils/src/test/java/io/scalecube/utils/IdGeneratorTest.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..8990c208 --- /dev/null +++ b/.gitignore @@ -0,0 +1,31 @@ +# Eclipse project files +.project +.classpath +.settings + +# IntelliJ IDEA project files and directories +*.iml +*.ipr +*.iws +.idea/ + +# Geany project file +.geany + +# KDevelop project file and directory +.kdev4/ +*.kdev4 + +# Build targets +/target +*/target + +# Report directories +/reports +*/reports + +# Mac-specific directory that no other operating system needs. +.DS_Store + +# JVM crash logs +hs_err_pid*.log \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..fbe8bab1 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,27 @@ +language: java +addons: + apt: + packages: + - libxml-xpath-perl +sudo: required +jdk: +- oraclejdk8 +notifications: + email: false + slack: + secure: FwMHwOFfe7lNgwo+ID+mIX4w9A1T0BJEthGM2aSDH9iqnlFdYKiequgzIFAfQHuzHIzjZsEF6y12uYPANrZvTa+3uqMStMqrWtkeNpGTXew5fnZLztgcJwXsPwbkDpNJcctXV++0TwWheVXBF9+mYbiXdQsimfbhqTu5yxnUG62UEhN/W+HlyAvf1TrMogCWABns00Iy9rAZjd9Ry5fGF1Fu1r3hOryipvoiVfnQJIn0+X+a6wAIyFddzA1p/GxS6/BOeT3dtXCBc+/pGjsHW9TWJoEkpxJ2AiZohAR+fTVW8e38/O6zIpSMOYSSCz8AGjWNfHCR0RjsDA0A6Ahsooon7KNx9qaK1X/tUtfQvE3c1HhwAEqz7vFE4OTSeY93f0Xiy/uLu7j21SGTl+CuYK8osbqFGW+pIwm8bu7oRRlHHDGZniWqGZkmvEPOLOKyk8/J7iYzJExgAZfyHawUUK0fYh8t/yxCXVyOz9G+s5gq6AX3rEvDp12kJzfUzxZX1XB4J6G2k9vlez5C9zghCkiSWSB5QuptafpauoKMdWmuGaftDdDHO+L0ZR14L08ctioPkQnVIWm7BNHUgGWiX2HFo9nsuhZ3+jzNUnt6hl7f3r4j2FAsYeQCy0Vfb51LhdVojIwPtpAbi68Uj3t2dIcA1nc357zfRvDCe+zORW4= +before_install: +- "./src/main/scripts/ci/before-install.sh" +- "./src/main/scripts/cd/before-deploy.sh" +after_success: +- java -jar ~/codacy-coverage-reporter-assembly.jar -l Java -r ./services/target/site/jacoco/jacoco.xml +deploy: +- provider: script + script: mvn -P release deploy -DskipTests=true -B -V -s travis-settings.xml + on: + branch: develop +- provider: script + script: + - "./src/main/scripts/cd/deploy.sh" + on: + branch: master diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 00000000..26b1d3e8 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,64 @@ +# Changes + +## 1.0.9 / SNAPSHOT + +* fix issue #291 #289 listen method duplicate stream responses + +## 1.0.8 / 2018-02-07 + +* Update ServiceRegistry api, expose 'methods' on ServiceInstance + +## 1.0.7 / 2017-12-05 + +* Introduce ServiceCall.invokeAll service endpoints +* Metrics added to ServiceCall request / response / error rates KPIs +* Add member host/port overrides on member config + + Shout-out: +- Mike Barker on first contribution. + + +## 1.0.6 / 2017-09-20 + +* Supports graceful shutdown of cluster member by spreading leaving notification +* Support rx.Observables as service response type +* Update dependencies introduce performance improvements + +## 1.0.5 / 2017-08-05 + +* Add @Inject annotation for services without special params +* Improve ClusterConfig API + +## 1.0.4 / 2017-06-05 + +* Fixed proxy api resolve issue when it call inside the service +* Use separate transport for services and use cluster for discovery only +* Fix issue with binding transport to IPv6 address +* Add @ServiceProxy inject annotation +* Reduce tech debt and improve testing + +## 1.0.3 / 2016-12-28 + +* Improve ClusterConfig API +* Fix cluster backpressure issue and gossip storm issue on initial sync +* Add possibility to change Cluster metadata dynamically after cluster started +* Move cluster join methods to Cluster interface and make ClusterImpl package private +* Reduce GC-load on message send via using Netty's voidPromise +* Transport API improvements +* Stabilize membership protocol +* Improve reliability of GossipProtocol under adverse conditions (message loss 50%) +* Support service dispatchers +* Do not create observable filter on each call to cluster's listen method +* Fix issue when starting many nodes at once throw exception address already in use + +## 1.0.2 / 2016-12-05 + +* Support service tags + +## 1.0.1 / 2016-11-22 + +* Fix backpressure issue + +## 1.0.0 / 2016-11-13 + +* Initial stable release diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..a154c7b5 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,46 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at info@scalecube.io. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 00000000..72e6d386 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,20 @@ +# Setting up development environment + +## Install the necessary build tools +You require to have latest stable [Oracle JDK 7](http://java.oracle.com/), latest stable +[Apache Maven](http://maven.apache.org/) and [Git](http://git-scm.com/) installed on your machine. + +## Set up IntelliJ IDEA +ScaleCube project team uses [IntelliJ IDEA](http://www.jetbrains.com/idea/) as the primary IDE, although we are fine +with using other development environments as long as you adhere to our coding style. + +### Code style +ScaleCube project team uses [Google code style](http://google.github.io/styleguide/javaguide.html) with +next modifications: +* Maximum line length is 120 +* Special package in import order is io.scalecube. + +We use [Eclipse code formatter plugin] (https://github.com/krasa/EclipseCodeFormatter#instructions). +Download [this code style configuration](https://github.com/scalecube/scalecube/blob/master/eclipse-java-google-style.xml) +and [import order configuration](https://github.com/scalecube/scalecube/blob/master/style.importorder) and configure +your IDEA plugin with those settings. Configuration is also available in the root project folder. diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 00000000..db6e13d1 --- /dev/null +++ b/README.md @@ -0,0 +1,187 @@ +# ScaleCube + +[![Build Status](https://travis-ci.org/scalecube/scalecube.svg?branch=master)](https://travis-ci.org/scalecube/scalecube) +[![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.scalecube/scalecube-cluster/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.scalecube/scalecube-cluster) +[![Codacy Badge](https://api.codacy.com/project/badge/Coverage/7a02aba38e5d4744ae3e3100a6b542a5)](https://www.codacy.com/app/ronenn/scalecube?utm_source=github.com&utm_medium=referral&utm_content=scalecube/scalecube&utm_campaign=Badge_Coverage) +[![Codacy Badge](https://api.codacy.com/project/badge/Grade/a162edec5ca347ef87db19320e41138a)](https://www.codacy.com/app/ScaleCube/scalecube?utm_source=github.com&utm_medium=referral&utm_content=scalecube/scalecube&utm_campaign=Badge_Grade) +[![Join the chat at https://gitter.im/scalecube/Lobby](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/scalecube/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Twitter URL](https://img.shields.io/twitter/url/https/twitter.com/fold_left.svg?style=social&label=Follow%20%40ScaleCube)](https://twitter.com/scalecube) + +## Welcome to scalecube! if you are new to scalecube: +[Please read ScaleCube-services Motivations and vision](https://github.com/scalecube/scalecube/wiki) + +## Overview +ScaleCube project provides the tools to develop, test and scale distributed components in a cluster with ease. +it provides a general purpose cluster-membership library based on [SWIM Membership protocol](http://www.cs.cornell.edu/~asdas/research/dsn02-swim.pdf) with gossip protocol as improvement. scalecube-cluster is Embeddable and can be used by any cluster-aware application that requires cluster membership, node discovery, failure-detection and gossip style of communication across member nodes. based on that unique properties scalecube implements aditional seperated module for microservices coordination library that features service discovery, fault-tolerance at scale. + +The project focuses on ensuring that your application realizes the full potential of the [Reactive Manifesto](http://www.reactivemanifesto.org/), +while delivering a high productivity development environment, and seamless production deployment experience. + +Web Site: [http://scalecube.io](http://scalecube.io/) + +## Features + +ScaleCube is designed as an embeddable library for the Java VM. It is built in a modular way where each module independently +implements a useful set of functionality and provides foundation for higher level features by abstracting away lower level concerns. + +Next is described modules in the top to bottom order from the higher level features to the low level components. + +### MICROSERVICES + +ScaleCube Services provides a low latency Reactive Microservices library for peer-to-peer service registry and discovery +based on gossip protocol ad without single point-of-failure or bottlenecks. + +ScaleCube Services Features: + +* Provision and interconnect microservices as a unified system (cluster) +* Async RPC with java-8 CompleteableFutures support +* Reactive Streams support with RxJava. +* No single-point-of-failure or single-point-of-bottleneck +* Cluster aware and distributed +* Modular, flexible deployment models and topology +* Zero configuration, automatic peer-to-peer service discovery using gossip +* Simple non-blocking, asynchronous programming model +* Resilient due to failure detection, fault tolerance, and elasticity +* Routing and balancing strategies for both stateless and stateful services +* Low latency and high throughput +* Takes advantage of the JVM and scales over available cores +* Embeddable to existing Java applications +* Message Driven based on google-protocol-buffers +* Natural Circuit-Breaker due to tight integration with scalecube-cluster failure detector. +* Support Service instance tagging. + +User Guide: + +* [Services Overview](http://scalecube.io/services.html) +* [Defining Services](http://scalecube.io/user-reference/services/DefineService.html) +* [Implementing services](http://scalecube.io/user-reference/services/ServiceImplementation.html) +* [Provisioning Clustered Services](http://scalecube.io/user-reference/services/ProvisionClusterServices.html) +* [Consuming services](http://scalecube.io/user-reference/services/ConsumingServices.html) + + +### CLUSTER + +ScaleCube Cluster is a lightweight decentralized cluster membership, failure detection, and gossip protocol library. +It provides an implementation of [SWIM](http://www.cs.cornell.edu/~asdas/research/dsn02-swim.pdf) cluster membership protocol for Java VM. +It is an efficient and scalable weakly-consistent distributed group membership protocol based on gossip-style communication between the +nodes in the cluster. Read my [blog post](http://www.antonkharenko.com/2015/09/swim-distributed-group-membership.html) with distilled +notes on the SWIM paper for more details. + +It is using a [random-probing failure detection algorithm](http://www.antonkharenko.com/2015/08/scalable-and-efficient-distributed.html) which provides +a uniform expected network load at all members. +The worst-case network load is linear O(n) for overall network produced by running algorithm on all nodes and constant network +load at one particular member independent from the size of the cluster. + +ScaleCube Cluster implements all improvements described at original SWIM algorithm paper, such as gossip-style dissemination, suspicion mechanism +and time-bounded strong completeness of failure detector algorithm. In addition to that we have introduced support of additional SYNC mechanism +in order to improve recovery of the cluster from network partitioning events. + +Using ScaleCube Cluster as simple as few lines of code: + +``` java +// Start cluster node Alice as a seed node of the cluster, listen and print all incoming messages +Cluster alice = Cluster.joinAwait(); +alice.listen().subscribe(msg -> System.out.println("Alice received: " + msg.data())); + +// Join cluster node Bob to cluster with Alice, listen and print all incoming messages +Cluster bob = Cluster.joinAwait(alice.address()); +bob.listen().subscribe(msg -> System.out.println("Bob received: " + msg.data())); + +// Join cluster node Carol to cluster with Alice (and Bob which is resolved via Alice) +Cluster carol = Cluster.joinAwait(alice.address()); + +// Send from Carol greeting message to all other cluster members (which is Alice and Bob) +carol.otherMembers().forEach(member -> carol.send(member, Message.fromData("Greetings from Carol"))); +``` + +You are welcome to explore javadoc documentation on cluster API and examples module for more advanced use cases. + +### TRANSPORT + +ScaleCube Transport is a network communication layer which provides high throughput and low latency peer-to-peer messaging. +It is based on [Netty](http://netty.io/) asynchronous networking framework and is using [RxJava](https://github.com/ReactiveX/RxJava) +in order to provide convenient reactive API on top of network handlers pipelines. + +Using ScaleCube Transport as simple as few lines of code: + +``` java +// Bind first transport to port 5000 +TransportConfig config1 = TransportConfig.builder().port(5000).build(); +Transport transport1 = Transport.bindAwait(config1); + +// Make first transport to listen and print all incoming messages +transport1.listen().subscribe(System.out::println); + +// Get 'host:port' address of the first transport +Address address1 = transport1.address(); + +// Bind second transport on available port and send message to the first transport +Transport transport2 = Transport.bindAwait(); +transport2.send(address1, Message.fromData("Hello World")); +``` + +You are welcome to explore javadoc documentation on transport API for more advanced use cases. + +## Support + +For improvement requests, bugs and discussions please use the [GitHub Issues](https://github.com/scalecube/scalecube/issues) +or chat with us to get support on [Gitter](https://gitter.im/scalecube/Lobby). + +You are more then welcome to join us or just show your support by granting us a small star :) + +## Maven + +Binaries and dependency information for Maven can be found at +[http://search.maven.org](http://search.maven.org/#search%7Cga%7C1%7Cio.scalecube.scalecube). + +To add a dependency on ScaleCube Services using Maven, use the following: + +``` xml + + io.scalecube + scalecube-services + x.y.z + +``` + +To add a dependency on ScaleCube Cluster using Maven, use the following: + +``` xml + + io.scalecube + scalecube-cluster + x.y.z + +``` + +To add a dependency on ScaleCube Transport using Maven, use the following: + +``` xml + + io.scalecube + scalecube-transport + x.y.z + +``` + +## Contributing +* Follow/Star us on github. +* Fork (and then git clone https://github.com/--your-username-here--/scalecube.git). +* Create a branch (git checkout -b branch_name). +* Commit your changes (git commit -am "Description of contribution"). +* Push the branch (git push origin branch_name). +* Open a Pull Request. +* Thank you for your contribution! Wait for a response... + +## References +* Anton Kharenko + + [blog](http://www.antonkharenko.com/) + +* Ronen Nachmias + + [posts](https://www.linkedin.com/today/author/ronenhm?trk=pprof-feed) + +## License + +[Apache License, Version 2.0](https://github.com/scalecube/scalecube/blob/master/LICENSE.txt) diff --git a/benchmark/pom.xml b/benchmark/pom.xml new file mode 100644 index 00000000..bda8ec74 --- /dev/null +++ b/benchmark/pom.xml @@ -0,0 +1,77 @@ + + + 4.0.0 + + io.scalecube + scalecube-parent + 1.0.9-SNAPSHOT + + scalecube-benchmark + ScaleCube/Benchmark + + + scalecube-benchmark + 1.4.1 + + + + + ${project.groupId} + scalecube-cluster + ${project.version} + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.2 + + + package + + shade + + + ${uberjar.name} + + + org.openjdk.jmh.Main + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + diff --git a/benchmark/src/main/java/cache/CacheReadBenchmark.java b/benchmark/src/main/java/cache/CacheReadBenchmark.java new file mode 100644 index 00000000..fb835ede --- /dev/null +++ b/benchmark/src/main/java/cache/CacheReadBenchmark.java @@ -0,0 +1,65 @@ +package cache; + +import com.google.common.base.Optional; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.concurrent.TimeUnit; + +import javax.annotation.Nonnull; + +@Fork(2) +@State(Scope.Benchmark) +@Threads(4) +@Warmup(iterations = 5) +@Measurement(iterations = 5) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +public class CacheReadBenchmark { + + static final String[] GENERIC_ARGS = new String[] {"java.lang.String", "java.lang.Long", "java.lang.Integer", + "java.lang.Short", "java.lang.Character", "java.lang.Byte", "io.scalecube.transport.protocol.MessageSchema",}; + + static final String[] GENERIC_ARGS_CACHE_MISS = new String[] {"java.lang.String", "xyz", "java.lang.Integer", "abc", + "java.lang.Character", "qwerty", "io.scalecube.transport.protocol.MessageSchema",}; + + LoadingCache> guavaCache = CacheBuilder.newBuilder().build( + new CacheLoader>() { + @Override + public Optional load(@Nonnull String className) { + try { + Class dataClass = Class.forName(className); + return Optional.of(dataClass); + } catch (ClassNotFoundException e) { + return Optional.absent(); + } + } + }); + + @Benchmark + public void readFromGuavaCache() { + for (String arg : GENERIC_ARGS) { + guavaCache.getUnchecked(arg); + } + } + + @Benchmark + public void readFromGuavaWithCacheMiss() { + for (String arg : GENERIC_ARGS_CACHE_MISS) { + guavaCache.getUnchecked(arg); + } + } + +} diff --git a/benchmark/src/main/java/io/scalecube/cluster/gossip/ProtostuffGossipBenchmark.java b/benchmark/src/main/java/io/scalecube/cluster/gossip/ProtostuffGossipBenchmark.java new file mode 100644 index 00000000..e8c4100c --- /dev/null +++ b/benchmark/src/main/java/io/scalecube/cluster/gossip/ProtostuffGossipBenchmark.java @@ -0,0 +1,95 @@ +package io.scalecube.cluster.gossip; + +import io.scalecube.cluster.Member; +import io.scalecube.transport.Address; +import io.scalecube.transport.Message; +import io.scalecube.transport.MessageCodec; + +import com.google.common.collect.ImmutableList; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +@Fork(2) +@State(Scope.Thread) +@Threads(4) +@Warmup(iterations = 5) +@Measurement(iterations = 5) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +public class ProtostuffGossipBenchmark { + static final String PAYLOAD = "Tl4KqQXZ5aMiIw29"; + static final String PAYLOAD_X32 = + "tI8Ppp8ShAp7IEDSFV1IgZCDKH2WyLI0NeSNc9oQOhQZXcHOzktuQBQmT5EGNitohtS1LShvdHgtAtWRz" + + "e03rXCPM1dhAGOgPjIFBFA2fQGTOlh8icxKVx8jwyFEqqehmIubcvfEFYmsb1QZgTgu2wdB4XxpTsspxUq4pc0O124AsU61W9tw7L" + + "ZWlhij1EA44sPrfjgW9kIZCEf8HL2zOXMUyNqRdzXD2VlxerrW7X8jcbYSCn5jkkXsc0E5Hf7P5DcuEcASrGH3ZhaGv94pnSneX9BJ" + + "rRIQxtmAdPPUXzVXrKw0mYcoB6Ye8mWuvVUFOO1Io6NJk0q2HHKST0FQVQo6mVEYKm8geTI6WphB4uRiQ6ksk6zOrJXmwQ6ssJIPYR" + + "RF2Mx9EDR9OMhYw6hoijdfgd23EXz8WkTkYz42kQkK99rNxyIXVMVyRPzLHBclYaYKlcmoN8f7hq6aiv3VxlPPchZ6xWmjOlGJY9P7" + + "nINtChd2spMUkhAeznajS4VW"; + + Message gossipReq; + ByteBuf gossipReqSer; + ByteBuf bbGossipReq; + + Message gossipReqx32; + ByteBuf gossipReqx32Ser; + ByteBuf bbGossipReqx32; + + /** + * Setup benchmark. + */ + @Setup + public void setup() { + List gossips = ImmutableList.of(new Gossip("ABCDEFGH_0", Message.fromData(PAYLOAD_X32))); + Member from = new Member("0", Address.from("localhost:1234")); + gossipReq = Message.fromData(new GossipRequest(gossips, from.id())); + MessageCodec.serialize(gossipReq, gossipReqSer = Unpooled.buffer(1024)); + bbGossipReq = Unpooled.buffer(1024); + System.err.println("### gossipReqSer=" + gossipReqSer); + + List list32 = new ArrayList<>(); + for (int i = 0; i < 32; i++) { + list32.add(new Gossip("ABCDEFGH_" + i, Message.fromData(PAYLOAD))); + } + gossipReqx32 = Message.fromData(new GossipRequest(list32, from.id())); + MessageCodec.serialize(gossipReqx32, gossipReqx32Ser = Unpooled.buffer(1024)); + bbGossipReqx32 = Unpooled.buffer(1024); + System.err.println("### gossipReqx32Ser=" + gossipReqx32Ser); + } + + @Benchmark + public void ser() { + MessageCodec.serialize(gossipReq, bbGossipReq.resetWriterIndex()); + } + + @Benchmark + public void deser() { + MessageCodec.deserialize(gossipReqSer.resetReaderIndex()); + } + + @Benchmark + public void ser_x32() { + MessageCodec.serialize(gossipReqx32, bbGossipReqx32.resetWriterIndex()); + } + + @Benchmark + public void deser_x32() { + MessageCodec.deserialize(gossipReqx32Ser.resetReaderIndex()); + } +} diff --git a/benchmark/src/main/java/io/scalecube/transport/ProtostuffMessageBenchmark.java b/benchmark/src/main/java/io/scalecube/transport/ProtostuffMessageBenchmark.java new file mode 100644 index 00000000..d7867471 --- /dev/null +++ b/benchmark/src/main/java/io/scalecube/transport/ProtostuffMessageBenchmark.java @@ -0,0 +1,77 @@ +package io.scalecube.transport; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.concurrent.TimeUnit; + +@Fork(2) +@State(Scope.Thread) +@Threads(4) +@Warmup(iterations = 5) +@Measurement(iterations = 5) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +public class ProtostuffMessageBenchmark { + static final String PAYLOAD = "SgUKzpnrt8ArR9jz"; + static final String PAYLOAD_X16 = "M82qEG06ucgawpZ89PgJcBhWiDIOZgEgz8o42ZuBrXVEXlUNmXfPdY1BOh4UbbwxTuTNAeyosxlZjDOf" + + "EfxPKPM2Al5CVkpg5175hzLBV5afcocm52JKwDvgSKVkoMzvnVWIQfjeAgGIERBgJ7a63mGygKDQS4moeHryedn68mmzNHGYbSqp7PIb6Rb" + + "n8SgT1hSOATWBReLA4ZPqfGUV0miIgOU90EYXffu9aT4cc9V8rsz3q4W8ibMsxq1JMsB6"; + + Message msg; + ByteBuf msgSer; + ByteBuf bbMsg; + + Message msgx16; + ByteBuf msgx16Ser; + ByteBuf bbMsgx16; + + /** + * Setup benchmark. + */ + @Setup + public void setup() { + + msg = Message.fromData(PAYLOAD); + MessageCodec.serialize(msg, msgSer = Unpooled.buffer(1024)); + bbMsg = Unpooled.buffer(1024); + System.err.println("### msgSer=" + msgSer); + + msgx16 = Message.fromData(PAYLOAD_X16); + MessageCodec.serialize(msgx16, msgx16Ser = Unpooled.buffer(1024)); + bbMsgx16 = Unpooled.buffer(1024); + System.err.println("### msgx16Ser=" + msgx16Ser); + } + + @Benchmark + public void ser() { + MessageCodec.serialize(msg, bbMsg.resetWriterIndex()); + } + + @Benchmark + public void deser() { + MessageCodec.deserialize(msgSer.resetReaderIndex()); + } + + @Benchmark + public void ser_x16() { + MessageCodec.serialize(msgx16, bbMsgx16.resetWriterIndex()); + } + + @Benchmark + public void deser_x16() { + MessageCodec.deserialize(msgx16Ser.resetReaderIndex()); + } +} diff --git a/benchmark/test/pom.xml b/benchmark/test/pom.xml new file mode 100644 index 00000000..5314922b --- /dev/null +++ b/benchmark/test/pom.xml @@ -0,0 +1,171 @@ + + + + 4.0.0 + + org.sample + test + 1.0 + jar + + JMH benchmark sample: Java + + + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + + + UTF-8 + + + 1.20 + + + 1.8 + + + benchmarks + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + ${javac.target} + ${javac.target} + ${javac.target} + + + + org.apache.maven.plugins + maven-shade-plugin + 2.2 + + + package + + shade + + + ${uberjar.name} + + + org.openjdk.jmh.Main + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + maven-clean-plugin + 2.5 + + + maven-deploy-plugin + 2.8.1 + + + maven-install-plugin + 2.5.1 + + + maven-jar-plugin + 2.4 + + + maven-javadoc-plugin + 2.9.1 + + + maven-resources-plugin + 2.6 + + + maven-site-plugin + 3.3 + + + maven-source-plugin + 2.2.1 + + + maven-surefire-plugin + 2.17 + + + + + + diff --git a/checkstyle-suppression.xml b/checkstyle-suppression.xml new file mode 100644 index 00000000..a0f28a6d --- /dev/null +++ b/checkstyle-suppression.xml @@ -0,0 +1,10 @@ + + + + + + + + \ No newline at end of file diff --git a/checkstyle.xml b/checkstyle.xml new file mode 100644 index 00000000..cae1c560 --- /dev/null +++ b/checkstyle.xml @@ -0,0 +1,208 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cluster/pom.xml b/cluster/pom.xml new file mode 100644 index 00000000..cb4c2b65 --- /dev/null +++ b/cluster/pom.xml @@ -0,0 +1,26 @@ + + + 4.0.0 + + io.scalecube + scalecube-parent + 1.0.9-SNAPSHOT + + scalecube-cluster + ScaleCube/Cluster + + + + ${project.groupId} + scalecube-transport + ${project.version} + + + ${project.groupId} + scalecube-testlib + ${project.version} + test + + + + diff --git a/cluster/src/main/java/io/scalecube/cluster/Cluster.java b/cluster/src/main/java/io/scalecube/cluster/Cluster.java new file mode 100644 index 00000000..ff84c1f6 --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/Cluster.java @@ -0,0 +1,202 @@ +package io.scalecube.cluster; + +import io.scalecube.cluster.membership.MembershipEvent; +import io.scalecube.transport.Address; +import io.scalecube.transport.Message; +import io.scalecube.transport.NetworkEmulator; + +import com.google.common.base.Throwables; + +import rx.Observable; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +import javax.annotation.Nonnull; + +/** + * Facade cluster interface which provides API to interact with cluster members. + * + * @author Anton Kharenko + */ +public interface Cluster { + + /** + * Init cluster instance and join cluster synchronously. + */ + static Cluster joinAwait() { + try { + return join().get(); + } catch (Exception e) { + throw Throwables.propagate(Throwables.getRootCause(e)); + } + } + + /** + * Init cluster instance with the given seed members and join cluster synchronously. + */ + static Cluster joinAwait(Address... seedMembers) { + try { + return join(seedMembers).get(); + } catch (Exception e) { + throw Throwables.propagate(Throwables.getRootCause(e)); + } + } + + /** + * Init cluster instance with the given metadata and seed members and join cluster synchronously. + */ + static Cluster joinAwait(Map metadata, Address... seedMembers) { + try { + return join(metadata, seedMembers).get(); + } catch (Exception e) { + throw Throwables.propagate(Throwables.getRootCause(e)); + } + } + + /** + * Init cluster instance with the given configuration and join cluster synchronously. + */ + static Cluster joinAwait(ClusterConfig config) { + try { + return join(config).get(); + } catch (Exception e) { + throw Throwables.propagate(Throwables.getRootCause(e)); + } + } + + /** + * Init cluster instance and join cluster asynchronously. + */ + static CompletableFuture join() { + return join(ClusterConfig.defaultConfig()); + } + + /** + * Init cluster instance with the given seed members and join cluster asynchronously. + */ + static CompletableFuture join(Address... seedMembers) { + ClusterConfig config = ClusterConfig.builder() + .seedMembers(seedMembers) + .build(); + return join(config); + } + + /** + * Init cluster instance with the given metadata and seed members and join cluster synchronously. + */ + static CompletableFuture join(Map metadata, Address... seedMembers) { + ClusterConfig config = ClusterConfig.builder() + .seedMembers(Arrays.asList(seedMembers)) + .metadata(metadata) + .build(); + return join(config); + } + + /** + * Init cluster instance with the given configuration and join cluster synchronously. + */ + static CompletableFuture join(final ClusterConfig config) { + return new ClusterImpl(config).join0(); + } + + /** + * Returns local listen {@link Address} of this cluster instance. + */ + Address address(); + + void send(Member member, Message message); + + void send(Member member, Message message, CompletableFuture promise); + + void send(Address address, Message message); + + void send(Address address, Message message, CompletableFuture promise); + + Observable listen(); + + /** + * Spreads given message between cluster members using gossiping protocol. + */ + CompletableFuture spreadGossip(Message message); + + /** + * Listens for gossips from other cluster members. + */ + Observable listenGossips(); + + /** + * Returns local cluster member which corresponds to this cluster instance. + */ + Member member(); + + /** + * Returns cluster member with given id or null if no member with such id exists at joined cluster. + */ + Optional member(String id); + + /** + * Returns cluster member by given address or null if no member with such address exists at joined cluster. + */ + Optional member(Address address); + + /** + * Returns list of all members of the joined cluster. This will include all cluster members including local member. + */ + Collection members(); + + /** + * Returns list of all cluster members of the joined cluster excluding local member. + */ + Collection otherMembers(); + + /** + * Updates local member metadata with the given metadata map. Metadata is updated asynchronously and results in a + * membership update event for local member once it is updated locally. Information about new metadata is disseminated + * to other nodes of the cluster with a weekly-consistent guarantees. + * + * @param metadata new metadata + */ + void updateMetadata(Map metadata); + + /** + * Updates single key-value pair of local member's metadata. This is a shortcut method and anyway update will result + * in a full metadata update. In case if you need to update several metadata property together it is recommended to + * use {@link #updateMetadata(Map)}. + * + * @param key metadata key to update + * @param value metadata value to update + */ + void updateMetadataProperty(String key, String value); + + /** + * Listen changes in cluster membership. + */ + Observable listenMembership(); + + /** + * Member notifies other members of the cluster about leaving and gracefully shutdown and free occupied resources. + * + * @return Listenable future which is completed once graceful shutdown is finished. + */ + CompletableFuture shutdown(); + + /** + * Check if cluster instance has been shut down. + * + * @return Returns true if cluster instance has been shut down; false otherwise. + */ + boolean isShutdown(); + + /** + * Returns network emulator associated with this instance of cluster. It always returns non null instance even if + * network emulator is disabled by transport config. In case when network emulator is disable all calls to network + * emulator instance will result in no operation. + */ + @Nonnull + NetworkEmulator networkEmulator(); + +} diff --git a/cluster/src/main/java/io/scalecube/cluster/ClusterConfig.java b/cluster/src/main/java/io/scalecube/cluster/ClusterConfig.java new file mode 100644 index 00000000..fa9a4a9e --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/ClusterConfig.java @@ -0,0 +1,392 @@ +package io.scalecube.cluster; + +import io.scalecube.cluster.fdetector.FailureDetectorConfig; +import io.scalecube.cluster.gossip.GossipConfig; +import io.scalecube.cluster.membership.MembershipConfig; +import io.scalecube.transport.Address; +import io.scalecube.transport.TransportConfig; + +import com.google.common.base.Preconditions; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Cluster configuration encapsulate settings needed cluster to create and successfully join. + * + * @see MembershipConfig + * @see FailureDetectorConfig + * @see GossipConfig + * @see TransportConfig + * + * @author Anton Kharenko + */ +public final class ClusterConfig implements FailureDetectorConfig, GossipConfig, MembershipConfig { + + // Default settings for LAN cluster + public static final String DEFAULT_SYNC_GROUP = "default"; + public static final int DEFAULT_SYNC_INTERVAL = 30_000; + public static final int DEFAULT_SYNC_TIMEOUT = 3_000; + public static final int DEFAULT_SUSPICION_MULT = 5; + public static final int DEFAULT_PING_INTERVAL = 1_000; + public static final int DEFAULT_PING_TIMEOUT = 500; + public static final int DEFAULT_PING_REQ_MEMBERS = 3; + public static final long DEFAULT_GOSSIP_INTERVAL = 200; + public static final int DEFAULT_GOSSIP_FANOUT = 3; + public static final int DEFAULT_GOSSIP_REPEAT_MULT = 3; + + // Default settings for WAN cluster (overrides default/LAN settings) + public static final int DEFAULT_WAN_SUSPICION_MULT = 6; + public static final int DEFAULT_WAN_SYNC_INTERVAL = 60_000; + public static final int DEFAULT_WAN_PING_TIMEOUT = 3_000; + public static final int DEFAULT_WAN_PING_INTERVAL = 5_000; + public static final int DEFAULT_WAN_GOSSIP_FANOUT = 4; + public static final int DEFAULT_WAN_CONNECT_TIMEOUT = 10_000; + + // Default settings for local cluster working via loopback interface (overrides default/LAN settings) + public static final int DEFAULT_LOCAL_SUSPICION_MULT = 3; + public static final int DEFAULT_LOCAL_SYNC_INTERVAL = 15_000; + public static final int DEFAULT_LOCAL_PING_TIMEOUT = 200; + public static final int DEFAULT_LOCAL_PING_INTERVAL = 1_000; + public static final int DEFAULT_LOCAL_GOSSIP_REPEAT_MULT = 2; + public static final int DEFAULT_LOCAL_PING_REQ_MEMBERS = 1; + public static final int DEFAULT_LOCAL_GOSSIP_INTERVAL = 100; + public static final int DEFAULT_LOCAL_CONNECT_TIMEOUT = 1_000; + + public static final String DEFAULT_MEMBER_HOST = null; + public static final Integer DEFAULT_MEMBER_PORT = null; + + private final List
seedMembers; + private final Map metadata; + private final int syncInterval; + private final int syncTimeout; + private final int suspicionMult; + private final String syncGroup; + + private final int pingInterval; + private final int pingTimeout; + private final int pingReqMembers; + + private final long gossipInterval; + private final int gossipFanout; + private final int gossipRepeatMult; + + private final TransportConfig transportConfig; + private final String memberHost; + private final Integer memberPort; + + private ClusterConfig(Builder builder) { + this.seedMembers = Collections.unmodifiableList(builder.seedMembers); + this.metadata = Collections.unmodifiableMap(builder.metadata); + this.syncInterval = builder.syncInterval; + this.syncTimeout = builder.syncTimeout; + this.syncGroup = builder.syncGroup; + this.suspicionMult = builder.suspicionMult; + + this.pingInterval = builder.pingInterval; + this.pingTimeout = builder.pingTimeout; + this.pingReqMembers = builder.pingReqMembers; + + this.gossipFanout = builder.gossipFanout; + this.gossipInterval = builder.gossipInterval; + this.gossipRepeatMult = builder.gossipRepeatMult; + + this.transportConfig = builder.transportConfigBuilder.build(); + this.memberHost = builder.memberHost; + this.memberPort = builder.memberPort; + } + + public static Builder builder() { + return new Builder(); + } + + public static ClusterConfig defaultConfig() { + return builder().build(); + } + + public static ClusterConfig defaultLanConfig() { + return defaultConfig(); + } + + /** + * Creates cluster config with default settings for cluster on WAN network. + */ + public static ClusterConfig defaultWanConfig() { + return builder() + .suspicionMult(DEFAULT_WAN_SUSPICION_MULT) + .syncInterval(DEFAULT_WAN_SYNC_INTERVAL) + .pingTimeout(DEFAULT_WAN_PING_TIMEOUT) + .pingInterval(DEFAULT_WAN_PING_INTERVAL) + .gossipFanout(DEFAULT_WAN_GOSSIP_FANOUT) + .connectTimeout(DEFAULT_WAN_CONNECT_TIMEOUT) + .build(); + } + + /** + * Creates cluster config with default settings for cluster on local loopback interface. + */ + public static ClusterConfig defaultLocalConfig() { + return builder() + .suspicionMult(DEFAULT_LOCAL_SUSPICION_MULT) + .syncInterval(DEFAULT_LOCAL_SYNC_INTERVAL) + .pingTimeout(DEFAULT_LOCAL_PING_TIMEOUT) + .pingInterval(DEFAULT_LOCAL_PING_INTERVAL) + .gossipRepeatMult(DEFAULT_LOCAL_GOSSIP_REPEAT_MULT) + .pingReqMembers(DEFAULT_LOCAL_PING_REQ_MEMBERS) + .gossipInterval(DEFAULT_LOCAL_GOSSIP_INTERVAL) + .connectTimeout(DEFAULT_LOCAL_CONNECT_TIMEOUT) + .build(); + } + + public List
getSeedMembers() { + return seedMembers; + } + + public Map getMetadata() { + return metadata; + } + + public int getSyncInterval() { + return syncInterval; + } + + public int getSyncTimeout() { + return syncTimeout; + } + + public int getSuspicionMult() { + return suspicionMult; + } + + public String getSyncGroup() { + return syncGroup; + } + + public int getPingInterval() { + return pingInterval; + } + + public int getPingTimeout() { + return pingTimeout; + } + + public int getPingReqMembers() { + return pingReqMembers; + } + + public int getGossipFanout() { + return gossipFanout; + } + + public long getGossipInterval() { + return gossipInterval; + } + + public int getGossipRepeatMult() { + return gossipRepeatMult; + } + + public TransportConfig getTransportConfig() { + return transportConfig; + } + + public String getMemberHost() { + return memberHost; + } + + public Integer getMemberPort() { + return memberPort; + } + + @Override + public String toString() { + return "ClusterConfig{seedMembers=" + seedMembers + + ", metadata=" + metadata + + ", syncInterval=" + syncInterval + + ", syncTimeout=" + syncTimeout + + ", suspicionMult=" + suspicionMult + + ", syncGroup='" + syncGroup + '\'' + + ", pingInterval=" + pingInterval + + ", pingTimeout=" + pingTimeout + + ", pingReqMembers=" + pingReqMembers + + ", gossipInterval=" + gossipInterval + + ", gossipFanout=" + gossipFanout + + ", gossipRepeatMult=" + gossipRepeatMult + + ", transportConfig=" + transportConfig + + ", memberHost=" + memberHost + + ", memberPort=" + memberPort + + '}'; + } + + public static final class Builder { + + private List
seedMembers = Collections.emptyList(); + private Map metadata = new HashMap<>(); + private int syncInterval = DEFAULT_SYNC_INTERVAL; + private int syncTimeout = DEFAULT_SYNC_TIMEOUT; + private String syncGroup = DEFAULT_SYNC_GROUP; + private int suspicionMult = DEFAULT_SUSPICION_MULT; + + private int pingInterval = DEFAULT_PING_INTERVAL; + private int pingTimeout = DEFAULT_PING_TIMEOUT; + private int pingReqMembers = DEFAULT_PING_REQ_MEMBERS; + + private long gossipInterval = DEFAULT_GOSSIP_INTERVAL; + private int gossipFanout = DEFAULT_GOSSIP_FANOUT; + private int gossipRepeatMult = DEFAULT_GOSSIP_REPEAT_MULT; + + private TransportConfig.Builder transportConfigBuilder = TransportConfig.builder(); + + private String memberHost = DEFAULT_MEMBER_HOST; + private Integer memberPort = DEFAULT_MEMBER_PORT; + + private Builder() {} + + public Builder metadata(Map metadata) { + this.metadata = new HashMap<>(metadata); + return this; + } + + public Builder addMetadata(String key, String value) { + this.metadata.put(key, value); + return this; + } + + public Builder addMetadata(Map metadata) { + this.metadata.putAll(metadata); + return this; + } + + public Builder seedMembers(Address... seedMembers) { + this.seedMembers = Arrays.asList(seedMembers); + return this; + } + + public Builder seedMembers(List
seedMembers) { + this.seedMembers = new ArrayList<>(seedMembers); + return this; + } + + public Builder syncInterval(int syncInterval) { + this.syncInterval = syncInterval; + return this; + } + + public Builder syncTimeout(int syncTimeout) { + this.syncTimeout = syncTimeout; + return this; + } + + public Builder suspicionMult(int suspicionMult) { + this.suspicionMult = suspicionMult; + return this; + } + + public Builder syncGroup(String syncGroup) { + this.syncGroup = syncGroup; + return this; + } + + public Builder pingInterval(int pingInterval) { + this.pingInterval = pingInterval; + return this; + } + + public Builder pingTimeout(int pingTimeout) { + this.pingTimeout = pingTimeout; + return this; + } + + public Builder pingReqMembers(int pingReqMembers) { + this.pingReqMembers = pingReqMembers; + return this; + } + + public Builder gossipInterval(long gossipInterval) { + this.gossipInterval = gossipInterval; + return this; + } + + public Builder gossipFanout(int gossipFanout) { + this.gossipFanout = gossipFanout; + return this; + } + + public Builder gossipRepeatMult(int gossipRepeatMult) { + this.gossipRepeatMult = gossipRepeatMult; + return this; + } + + /** + * Sets all transport config settings equal to provided transport config. + */ + public Builder transportConfig(TransportConfig transportConfig) { + this.transportConfigBuilder.fillFrom(transportConfig); + return this; + } + + public Builder listenAddress(String listenAddress) { + this.transportConfigBuilder.listenAddress(listenAddress); + return this; + } + + public Builder listenInterface(String listenInterface) { + this.transportConfigBuilder.listenInterface(listenInterface); + return this; + } + + public Builder port(int port) { + this.transportConfigBuilder.port(port); + return this; + } + + public Builder portAutoIncrement(boolean portAutoIncrement) { + this.transportConfigBuilder.portAutoIncrement(portAutoIncrement); + return this; + } + + public Builder connectTimeout(int connectTimeout) { + this.transportConfigBuilder.connectTimeout(connectTimeout); + return this; + } + + public Builder useNetworkEmulator(boolean useNetworkEmulator) { + this.transportConfigBuilder.useNetworkEmulator(useNetworkEmulator); + return this; + } + + /** + * Member address overrides. + */ + + /** + * Override the member host in cases when the transport address is not the address to be broadcast. + * @param memberHost Member host to broadcast + * @return this builder + */ + public Builder memberHost(String memberHost) { + this.memberHost = memberHost; + return this; + } + + /** + * Override the member port in cases when the transport port is not the post to be broadcast. + * @param memberPort Member port to broadcast + * @return this builder + */ + public Builder memberPort(Integer memberPort) { + this.memberPort = memberPort; + return this; + } + + + public ClusterConfig build() { + Preconditions.checkState(pingTimeout < pingInterval, "Ping timeout can't be bigger than ping interval"); + return new ClusterConfig(this); + } + } +} diff --git a/cluster/src/main/java/io/scalecube/cluster/ClusterImpl.java b/cluster/src/main/java/io/scalecube/cluster/ClusterImpl.java new file mode 100644 index 00000000..1fe2189f --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/ClusterImpl.java @@ -0,0 +1,242 @@ +package io.scalecube.cluster; + +import static com.google.common.base.Preconditions.checkNotNull; +import static io.scalecube.cluster.fdetector.FailureDetectorImpl.PING; +import static io.scalecube.cluster.fdetector.FailureDetectorImpl.PING_ACK; +import static io.scalecube.cluster.fdetector.FailureDetectorImpl.PING_REQ; +import static io.scalecube.cluster.gossip.GossipProtocolImpl.GOSSIP_REQ; +import static io.scalecube.cluster.membership.MembershipProtocolImpl.MEMBERSHIP_GOSSIP; +import static io.scalecube.cluster.membership.MembershipProtocolImpl.SYNC; +import static io.scalecube.cluster.membership.MembershipProtocolImpl.SYNC_ACK; + +import io.scalecube.cluster.fdetector.FailureDetectorImpl; +import io.scalecube.cluster.gossip.GossipProtocolImpl; +import io.scalecube.cluster.membership.MembershipEvent; +import io.scalecube.cluster.membership.MembershipProtocolImpl; +import io.scalecube.transport.Address; +import io.scalecube.transport.Message; +import io.scalecube.transport.NetworkEmulator; +import io.scalecube.transport.Transport; + +import com.google.common.collect.ImmutableSet; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import rx.Observable; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import javax.annotation.Nonnull; + +/** + * Cluster implementation. + * + * @author Anton Kharenko + */ +final class ClusterImpl implements Cluster { + + private static final Logger LOGGER = LoggerFactory.getLogger(ClusterImpl.class); + + private static final Set SYSTEM_MESSAGES = + ImmutableSet.of(PING, PING_REQ, PING_ACK, SYNC, SYNC_ACK, GOSSIP_REQ); + + private static final Set SYSTEM_GOSSIPS = ImmutableSet.of(MEMBERSHIP_GOSSIP); + + private final ClusterConfig config; + + private final ConcurrentMap members = new ConcurrentHashMap<>(); + private final ConcurrentMap memberAddressIndex = new ConcurrentHashMap<>(); + + // Cluster components + private Transport transport; + private FailureDetectorImpl failureDetector; + private GossipProtocolImpl gossip; + private MembershipProtocolImpl membership; + + private Observable messageObservable; + private Observable gossipObservable; + + public ClusterImpl(ClusterConfig config) { + checkNotNull(config); + this.config = config; + } + + public CompletableFuture join0() { + CompletableFuture transportFuture = Transport.bind(config.getTransportConfig()); + CompletableFuture clusterFuture = transportFuture.thenCompose(boundTransport -> { + transport = boundTransport; + messageObservable = transport.listen() + .filter(msg -> !SYSTEM_MESSAGES.contains(msg.qualifier())); // filter out system gossips + + membership = new MembershipProtocolImpl(transport, config); + gossip = new GossipProtocolImpl(transport, membership, config); + failureDetector = new FailureDetectorImpl(transport, membership, config); + membership.setFailureDetector(failureDetector); + membership.setGossipProtocol(gossip); + + Member localMember = membership.member(); + onMemberAdded(localMember); + membership.listen() + .filter(MembershipEvent::isAdded) + .map(MembershipEvent::member) + .subscribe(this::onMemberAdded, this::onError); + membership.listen() + .filter(MembershipEvent::isRemoved) + .map(MembershipEvent::member) + .subscribe(this::onMemberRemoved, this::onError); + membership.listen() + .filter(MembershipEvent::isUpdated) + .map(MembershipEvent::member) + .subscribe(this::onMemberUpdated, this::onError); + + failureDetector.start(); + gossip.start(); + gossipObservable = gossip.listen() + .filter(msg -> !SYSTEM_GOSSIPS.contains(msg.qualifier())); // filter out system gossips + return membership.start(); + }); + return clusterFuture.thenApply(aVoid -> ClusterImpl.this); + } + + private void onError(Throwable throwable) { + LOGGER.error("Received unexpected error: ", throwable); + } + + private void onMemberAdded(Member member) { + memberAddressIndex.put(member.address(), member.id()); + members.put(member.id(), member); + } + + private void onMemberRemoved(Member member) { + members.remove(member.id()); + memberAddressIndex.remove(member.address()); + } + + private void onMemberUpdated(Member member) { + members.put(member.id(), member); + } + + @Override + public Address address() { + return member().address(); + } + + @Override + public void send(Member member, Message message) { + transport.send(member.address(), message); + } + + @Override + public void send(Address address, Message message) { + transport.send(address, message); + } + + @Override + public void send(Member member, Message message, CompletableFuture promise) { + transport.send(member.address(), message, promise); + } + + @Override + public void send(Address address, Message message, CompletableFuture promise) { + transport.send(address, message, promise); + } + + @Override + public Observable listen() { + return messageObservable; + } + + @Override + public CompletableFuture spreadGossip(Message message) { + return gossip.spread(message); + } + + @Override + public Observable listenGossips() { + return gossipObservable; + } + + @Override + public Collection members() { + return Collections.unmodifiableCollection(members.values()); + } + + @Override + public Member member() { + return membership.member(); + } + + @Override + public Optional member(String id) { + return Optional.ofNullable(members.get(id)); + } + + @Override + public Optional member(Address address) { + return Optional.ofNullable(memberAddressIndex.get(address)) + .flatMap(memberId -> Optional.ofNullable(members.get(memberId))); + } + + @Override + public Collection otherMembers() { + ArrayList otherMembers = new ArrayList<>(members.values()); + otherMembers.remove(membership.member()); + return Collections.unmodifiableCollection(otherMembers); + } + + @Override + public void updateMetadata(Map metadata) { + membership.updateMetadata(metadata); + } + + @Override + public void updateMetadataProperty(String key, String value) { + membership.updateMetadataProperty(key, value); + } + + @Override + public Observable listenMembership() { + return membership.listen(); + } + + @Override + public CompletableFuture shutdown() { + LOGGER.info("Cluster member {} is shutting down...", membership.member()); + CompletableFuture transportStoppedFuture = new CompletableFuture<>(); + membership.leave() + .whenComplete((gossipId, error) -> { + LOGGER.info("Cluster member notified about his leaving and shutting down... {}", membership.member()); + + // stop algorithms + membership.stop(); + gossip.stop(); + failureDetector.stop(); + + // stop transport + transport.stop(transportStoppedFuture); + + LOGGER.info("Cluster member has shut down... {}", membership.member()); + }); + return transportStoppedFuture; + } + + @Nonnull + @Override + public NetworkEmulator networkEmulator() { + return transport.networkEmulator(); + } + + @Override + public boolean isShutdown() { + return this.transport.isStopped(); // since transport is the last component stopped on shutdown + } +} diff --git a/cluster/src/main/java/io/scalecube/cluster/ClusterMath.java b/cluster/src/main/java/io/scalecube/cluster/ClusterMath.java new file mode 100644 index 00000000..bcc6c0f9 --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/ClusterMath.java @@ -0,0 +1,60 @@ +package io.scalecube.cluster; + +/** + * Utility class which contains math computation on cluster properties. + * + * @author Anton Kharenko + */ +public final class ClusterMath { + + private ClusterMath() { + // Do not instantiate + } + + public static double gossipConvergencePercent(int fanout, int repeatMult, int clusterSize, double lossPercent) { + double msgLossProb = lossPercent / 100.0; + return gossipConvergenceProbability(fanout, repeatMult, clusterSize, msgLossProb) * 100; + } + + public static double gossipConvergenceProbability(int fanout, int repeatMult, int clusterSize, double loss) { + double fanoutWithLoss = (1.0 - loss) * fanout; + double spreadSize = clusterSize - Math.pow(clusterSize, -(fanoutWithLoss * repeatMult - 2)); + return spreadSize / clusterSize; + } + + public static int maxMessagesPerGossipTotal(int fanout, int repeatMult, int clusterSize) { + return clusterSize * maxMessagesPerGossipPerNode(fanout, repeatMult, clusterSize); + } + + public static int maxMessagesPerGossipPerNode(int fanout, int repeatMult, int clusterSize) { + return fanout * repeatMult * ceilLog2(clusterSize); + } + + public static long gossipDisseminationTime(int repeatMult, int clusterSize, long gossipInterval) { + return gossipPeriodsToSpread(repeatMult, clusterSize) * gossipInterval; + } + + public static long gossipTimeoutToSweep(int repeatMult, int clusterSize, long gossipInterval) { + return gossipPeriodsToSweep(repeatMult, clusterSize) * gossipInterval; + } + + public static int gossipPeriodsToSweep(int repeatMult, int clusterSize) { + int periodsToSpread = gossipPeriodsToSpread(repeatMult, clusterSize); + return 2 * (periodsToSpread + 1); + } + + public static int gossipPeriodsToSpread(int repeatMult, int clusterSize) { + return repeatMult * ceilLog2(clusterSize); + } + + public static long suspicionTimeout(int suspicionMult, int clusterSize, long pingInterval) { + return suspicionMult * ceilLog2(clusterSize) * pingInterval; + } + + /** + * Returns ceil(log2(n + 1)). + */ + public static int ceilLog2(int num) { + return 32 - Integer.numberOfLeadingZeros(num); + } +} diff --git a/cluster/src/main/java/io/scalecube/cluster/Member.java b/cluster/src/main/java/io/scalecube/cluster/Member.java new file mode 100644 index 00000000..876ad1c2 --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/Member.java @@ -0,0 +1,86 @@ +package io.scalecube.cluster; + +import static com.google.common.base.Preconditions.checkArgument; + +import io.scalecube.transport.Address; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import javax.annotation.concurrent.Immutable; + +/** + * Cluster member which represents node in the cluster and contains its id, address and metadata. + * + * @author Anton Kharenko + */ +@Immutable +public final class Member { + + private final String id; + private final Address address; + private final Map metadata; + + /** + * Create instance of cluster member with given id, address and empty metadata. + * + * @param id member id + * @param address address on which given member listens for incoming messages + */ + public Member(String id, Address address) { + this(id, address, Collections.emptyMap()); + } + + /** + * Create instance of cluster member with given parameters. + * + * @param id member id + * @param address address on which given member listens for incoming messages + * @param metadata member's metadata + */ + public Member(String id, Address address, Map metadata) { + checkArgument(id != null); + checkArgument(address != null); + this.id = id; + this.address = address; + this.metadata = metadata != null ? new HashMap<>(metadata) : Collections.emptyMap(); + } + + public String id() { + return id; + } + + public Address address() { + return address; + } + + public Map metadata() { + return Collections.unmodifiableMap(metadata); + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + if (that == null || getClass() != that.getClass()) { + return false; + } + Member member = (Member) that; + return Objects.equals(id, member.id) + && Objects.equals(address, member.address) + && Objects.equals(metadata, member.metadata); + } + + @Override + public int hashCode() { + return Objects.hash(id, address, metadata); + } + + @Override + public String toString() { + return id + "@" + address + (metadata.isEmpty() ? "" : metadata); + } +} diff --git a/cluster/src/main/java/io/scalecube/cluster/fdetector/FailureDetector.java b/cluster/src/main/java/io/scalecube/cluster/fdetector/FailureDetector.java new file mode 100644 index 00000000..484f239b --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/fdetector/FailureDetector.java @@ -0,0 +1,30 @@ +package io.scalecube.cluster.fdetector; + +import rx.Observable; + +/** + * Failure Detector component responsible for monitoring availability of other members in the cluster. This interface is + * supposed to be used internally as part cluster membership protocol. It doesn't specify that particular node is + * failed, but just provide information that either it is suspected or trusted at current moment of time. So it is up to + * cluster membership or other top level component to define when suspected member is actually failed. + * + * @author Anton Kharenko + */ +public interface FailureDetector { + + /** + * Starts running failure detection algorithm. After started it begins to receive and send ping messages. + */ + void start(); + + /** + * Stops running failure detection algorithm and releases occupied resources. + */ + void stop(); + + /** + * Listens for results of ping checks (ALIVE/SUSPECT) done periodically by failure detector. + */ + Observable listen(); + +} diff --git a/cluster/src/main/java/io/scalecube/cluster/fdetector/FailureDetectorConfig.java b/cluster/src/main/java/io/scalecube/cluster/fdetector/FailureDetectorConfig.java new file mode 100644 index 00000000..fcfaf09f --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/fdetector/FailureDetectorConfig.java @@ -0,0 +1,11 @@ +package io.scalecube.cluster.fdetector; + +public interface FailureDetectorConfig { + + int getPingInterval(); + + int getPingTimeout(); + + int getPingReqMembers(); + +} diff --git a/cluster/src/main/java/io/scalecube/cluster/fdetector/FailureDetectorEvent.java b/cluster/src/main/java/io/scalecube/cluster/fdetector/FailureDetectorEvent.java new file mode 100644 index 00000000..e4fc388d --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/fdetector/FailureDetectorEvent.java @@ -0,0 +1,33 @@ +package io.scalecube.cluster.fdetector; + +import io.scalecube.cluster.Member; +import io.scalecube.cluster.membership.MemberStatus; + +import javax.annotation.concurrent.Immutable; + +/** + * CLass contains result of ping check. */ +@Immutable +public final class FailureDetectorEvent { + + private final Member member; + private final MemberStatus status; + + FailureDetectorEvent(Member member, MemberStatus status) { + this.member = member; + this.status = status; + } + + public Member member() { + return member; + } + + public MemberStatus status() { + return status; + } + + @Override + public String toString() { + return "FailureDetectorEvent{member=" + member + ", status=" + status + '}'; + } +} diff --git a/cluster/src/main/java/io/scalecube/cluster/fdetector/FailureDetectorImpl.java b/cluster/src/main/java/io/scalecube/cluster/fdetector/FailureDetectorImpl.java new file mode 100644 index 00000000..88516c93 --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/fdetector/FailureDetectorImpl.java @@ -0,0 +1,388 @@ +package io.scalecube.cluster.fdetector; + +import static com.google.common.base.Preconditions.checkArgument; + +import io.scalecube.cluster.Member; +import io.scalecube.cluster.membership.MembershipProtocol; +import io.scalecube.cluster.membership.MemberStatus; +import io.scalecube.cluster.membership.MembershipEvent; +import io.scalecube.transport.Transport; +import io.scalecube.transport.Message; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import rx.Observable; +import rx.Scheduler; +import rx.Subscriber; +import rx.observers.Subscribers; +import rx.schedulers.Schedulers; +import rx.subjects.PublishSubject; +import rx.subjects.Subject; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; + +public final class FailureDetectorImpl implements FailureDetector { + + private static final Logger LOGGER = LoggerFactory.getLogger(FailureDetectorImpl.class); + + // Qualifiers + + public static final String PING = "sc/fdetector/ping"; + public static final String PING_REQ = "sc/fdetector/pingReq"; + public static final String PING_ACK = "sc/fdetector/pingAck"; + + // Injected + + private final Transport transport; + private final MembershipProtocol membership; + private final FailureDetectorConfig config; + + // State + + private long period = 0; + private List pingMembers = new ArrayList<>(); + private int pingMemberIndex = 0; // index for sequential ping member selection + + // Subscriptions + + private Subscriber onMemberAddedSubscriber; + private Subscriber onMemberRemovedSubscriber; + private Subscriber onMemberUpdatedSubscriber; + private Subscriber onPingRequestSubscriber; + private Subscriber onAskToPingRequestSubscriber; + private Subscriber onTransitPingAckRequestSubscriber; + + // Subject + + private Subject subject = + PublishSubject.create().toSerialized(); + + // Scheduled + + private final ScheduledExecutorService executor; + private final Scheduler scheduler; + private ScheduledFuture pingTask; + + /** + * Creates new instance of failure detector with given transport and settings. + * + * @param transport transport + * @param membership membership protocol + * @param config failure detector settings + */ + public FailureDetectorImpl(Transport transport, MembershipProtocol membership, FailureDetectorConfig config) { + checkArgument(transport != null); + checkArgument(membership != null); + checkArgument(config != null); + this.transport = transport; + this.membership = membership; + this.config = config; + String nameFormat = "sc-fdetector-" + Integer.toString(membership.member().address().port()); + this.executor = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat(nameFormat).setDaemon(true).build()); + this.scheduler = Schedulers.from(executor); + } + + /** + * NOTE: this method is for testing purpose only. + */ + Transport getTransport() { + return transport; + } + + @Override + public void start() { + onMemberAddedSubscriber = Subscribers.create(this::onMemberAdded, this::onError); + membership.listen().observeOn(scheduler) + .filter(MembershipEvent::isAdded) + .map(MembershipEvent::member) + .subscribe(onMemberAddedSubscriber); + + onMemberRemovedSubscriber = Subscribers.create(this::onMemberRemoved, this::onError); + membership.listen().observeOn(scheduler) + .filter(MembershipEvent::isRemoved) + .map(MembershipEvent::member) + .subscribe(onMemberRemovedSubscriber); + + onMemberUpdatedSubscriber = Subscribers.create(this::onMemberUpdated, this::onError); + membership.listen().observeOn(scheduler) + .filter(MembershipEvent::isUpdated) + .subscribe(onMemberUpdatedSubscriber); + + onPingRequestSubscriber = Subscribers.create(this::onPing, this::onError); + transport.listen().observeOn(scheduler) + .filter(this::isPing) + .subscribe(onPingRequestSubscriber); + + onAskToPingRequestSubscriber = Subscribers.create(this::onPingReq, this::onError); + transport.listen().observeOn(scheduler) + .filter(this::isPingReq) + .subscribe(onAskToPingRequestSubscriber); + + onTransitPingAckRequestSubscriber = Subscribers.create(this::onTransitPingAck, this::onError); + transport.listen().observeOn(scheduler) + .filter(this::isTransitPingAck) + .subscribe(onTransitPingAckRequestSubscriber); + + pingTask = executor.scheduleWithFixedDelay( + this::doPing, config.getPingInterval(), config.getPingInterval(), TimeUnit.MILLISECONDS); + } + + @Override + public void stop() { + // Stop accepting requests + if (onMemberAddedSubscriber != null) { + onMemberAddedSubscriber.unsubscribe(); + } + if (onMemberRemovedSubscriber != null) { + onMemberRemovedSubscriber.unsubscribe(); + } + if (onMemberUpdatedSubscriber != null) { + onMemberUpdatedSubscriber.unsubscribe(); + } + if (onPingRequestSubscriber != null) { + onPingRequestSubscriber.unsubscribe(); + } + if (onAskToPingRequestSubscriber != null) { + onAskToPingRequestSubscriber.unsubscribe(); + } + if (onTransitPingAckRequestSubscriber != null) { + onTransitPingAckRequestSubscriber.unsubscribe(); + } + + // Stop sending pings + if (pingTask != null) { + pingTask.cancel(true); + } + + // Shutdown executor + executor.shutdown(); + + // Stop publishing events + subject.onCompleted(); + } + + @Override + public Observable listen() { + return subject.onBackpressureDrop().asObservable(); + } + + // ================================================ + // ============== Action Methods ================== + // ================================================ + + private void doPing() { + // Increment period counter + period++; + + // Select ping member + Member pingMember = selectPingMember(); + if (pingMember == null) { + return; + } + + // Send ping + Member localMember = membership.member(); + String cid = localMember.id() + "-" + Long.toString(period); + PingData pingData = new PingData(localMember, pingMember); + Message pingMsg = Message.withData(pingData).qualifier(PING).correlationId(cid).build(); + try { + LOGGER.trace("Send Ping[{}] to {}", period, pingMember); + transport.listen().observeOn(scheduler) + .filter(this::isPingAck) + .filter(message -> cid.equals(message.correlationId())) + .take(1) + .timeout(config.getPingTimeout(), TimeUnit.MILLISECONDS, scheduler) + .subscribe( + message -> { + LOGGER.trace("Received PingAck[{}] from {}", period, pingMember); + publishPingResult(pingMember, MemberStatus.ALIVE); + }, + throwable -> { + LOGGER.trace("Timeout getting PingAck[{}] from {} within {} ms", + period, pingMember, config.getPingTimeout()); + doPingReq(pingMember, cid); + }); + transport.send(pingMember.address(), pingMsg); + } catch (Exception cause) { + LOGGER.error("Exception on sending Ping[{}] to {}: {}", period, pingMember, cause.getMessage(), cause); + } + } + + private void doPingReq(final Member pingMember, String cid) { + final int timeout = config.getPingInterval() - config.getPingTimeout(); + if (timeout <= 0) { + LOGGER.trace("No PingReq[{}] occurred, because no time left (pingInterval={}, pingTimeout={})", + period, config.getPingInterval(), config.getPingTimeout()); + publishPingResult(pingMember, MemberStatus.SUSPECT); + return; + } + + final List pingReqMembers = selectPingReqMembers(pingMember); + if (pingReqMembers.isEmpty()) { + LOGGER.trace("No PingReq[{}] occurred, because member selection is empty", period); + publishPingResult(pingMember, MemberStatus.SUSPECT); + return; + } + + Member localMember = membership.member(); + transport.listen().observeOn(scheduler) + .filter(this::isPingAck) + .filter(message -> cid.equals(message.correlationId())) + .take(1) + .timeout(timeout, TimeUnit.MILLISECONDS, scheduler) + .subscribe( + message -> { + LOGGER.trace("Received transit PingAck[{}] from {} to {}", period, message.sender(), pingMember); + publishPingResult(pingMember, MemberStatus.ALIVE); + }, + throwable -> { + LOGGER.trace("Timeout getting transit PingAck[{}] from {} to {} within {} ms", + period, pingReqMembers, pingMember, timeout); + publishPingResult(pingMember, MemberStatus.SUSPECT); + }); + + PingData pingReqData = new PingData(localMember, pingMember); + Message pingReqMsg = Message.withData(pingReqData).qualifier(PING_REQ).correlationId(cid).build(); + LOGGER.trace("Send PingReq[{}] to {} for {}", period, pingReqMembers, pingMember); + for (Member pingReqMember : pingReqMembers) { + transport.send(pingReqMember.address(), pingReqMsg); + } + } + + // ================================================ + // ============== Event Listeners ================= + // ================================================ + + private void onMemberAdded(Member member) { + // insert member into random positions + int size = pingMembers.size(); + int index = size > 0 ? ThreadLocalRandom.current().nextInt(size) : 0; + pingMembers.add(index, member); + } + + private void onMemberRemoved(Member member) { + pingMembers.remove(member); + } + + private void onMemberUpdated(MembershipEvent membershipEvent) { + int index = pingMembers.indexOf(membershipEvent.oldMember()); + if (index != -1) { // except local + pingMembers.set(index, membershipEvent.newMember()); + } + } + + /** + * Listens to PING message and answers with ACK. + */ + private void onPing(Message message) { + LOGGER.trace("Received Ping: {}", message); + PingData data = message.data(); + Member localMember = membership.member(); + if (!data.getTo().id().equals(localMember.id())) { + LOGGER.warn("Received Ping to {}, but local member is {}", data.getTo(), localMember); + return; + } + String correlationId = message.correlationId(); + Message ackMessage = Message.withData(data).qualifier(PING_ACK).correlationId(correlationId).build(); + LOGGER.trace("Send PingAck to {}", data.getFrom().address()); + transport.send(data.getFrom().address(), ackMessage); + } + + /** + * Listens to PING_REQ message and sends PING to requested cluster member. + */ + private void onPingReq(Message message) { + LOGGER.trace("Received PingReq: {}", message); + PingData data = message.data(); + Member target = data.getTo(); + Member originalIssuer = data.getFrom(); + String correlationId = message.correlationId(); + PingData pingReqData = new PingData(membership.member(), target, originalIssuer); + Message pingMessage = Message.withData(pingReqData).qualifier(PING).correlationId(correlationId).build(); + LOGGER.trace("Send transit Ping to {}", target.address()); + transport.send(target.address(), pingMessage); + } + + /** + * Listens to ACK with message containing ORIGINAL_ISSUER then converts message to plain ACK and sends it to + * ORIGINAL_ISSUER. + */ + private void onTransitPingAck(Message message) { + LOGGER.trace("Received transit PingAck: {}", message); + PingData data = message.data(); + Member target = data.getOriginalIssuer(); + String correlationId = message.correlationId(); + PingData originalAckData = new PingData(target, data.getTo()); + Message originalAckMessage = Message.withData(originalAckData) + .qualifier(PING_ACK) + .correlationId(correlationId) + .build(); + LOGGER.trace("Resend transit PingAck to {}", target.address()); + transport.send(target.address(), originalAckMessage); + } + + private void onError(Throwable throwable) { + LOGGER.error("Received unexpected error: ", throwable); + } + + // ================================================ + // ============== Helper Methods ================== + // ================================================ + + private Member selectPingMember() { + if (pingMembers.isEmpty()) { + return null; + } + if (pingMemberIndex >= pingMembers.size()) { + pingMemberIndex = 0; + Collections.shuffle(pingMembers); + } + return pingMembers.get(pingMemberIndex++); + } + + private List selectPingReqMembers(Member pingMember) { + if (config.getPingReqMembers() <= 0) { + return Collections.emptyList(); + } + List candidates = new ArrayList<>(pingMembers); + candidates.remove(pingMember); + if (candidates.isEmpty()) { + return Collections.emptyList(); + } + Collections.shuffle(candidates); + boolean selectAll = candidates.size() < config.getPingReqMembers(); + return selectAll ? candidates : candidates.subList(0, config.getPingReqMembers()); + } + + private void publishPingResult(Member member, MemberStatus status) { + LOGGER.debug("Member {} detected as {}", member, status); + subject.onNext(new FailureDetectorEvent(member, status)); + } + + private boolean isPing(Message message) { + return PING.equals(message.qualifier()); + } + + private boolean isPingReq(Message message) { + return PING_REQ.equals(message.qualifier()); + } + + private boolean isPingAck(Message message) { + return PING_ACK.equals(message.qualifier()) && message.data().getOriginalIssuer() == null; + } + + private boolean isTransitPingAck(Message message) { + return PING_ACK.equals(message.qualifier()) && message.data().getOriginalIssuer() != null; + } +} diff --git a/cluster/src/main/java/io/scalecube/cluster/fdetector/PingData.java b/cluster/src/main/java/io/scalecube/cluster/fdetector/PingData.java new file mode 100644 index 00000000..932953b0 --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/fdetector/PingData.java @@ -0,0 +1,50 @@ +package io.scalecube.cluster.fdetector; + +import io.scalecube.cluster.Member; + +import io.protostuff.Tag; + +/** DTO class. Supports FailureDetector messages (Ping, Ack, PingReq). */ +final class PingData { + /** Message's source address. */ + @Tag(1) + private final Member from; + /** Message's destination address. */ + @Tag(2) + private final Member to; + /** Address of member, who originally initiated ping sequence. */ + @Tag(3) + private final Member originalIssuer; + + public PingData(Member from, Member to) { + this.from = from; + this.to = to; + this.originalIssuer = null; + } + + public PingData(Member from, Member to, Member originalIssuer) { + this.from = from; + this.to = to; + this.originalIssuer = originalIssuer; + } + + public Member getFrom() { + return from; + } + + public Member getTo() { + return to; + } + + public Member getOriginalIssuer() { + return originalIssuer; + } + + @Override + public String toString() { + return "PingData{from=" + from + + ", to=" + to + + (originalIssuer != null ? ", originalIssuer=" + originalIssuer : "") + + '}'; + } +} diff --git a/cluster/src/main/java/io/scalecube/cluster/gossip/Gossip.java b/cluster/src/main/java/io/scalecube/cluster/gossip/Gossip.java new file mode 100644 index 00000000..27b969d3 --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/gossip/Gossip.java @@ -0,0 +1,54 @@ +package io.scalecube.cluster.gossip; + +import static com.google.common.base.Preconditions.checkArgument; + +import io.scalecube.transport.Message; + +import java.util.Objects; + +/** + * Data model for gossip, include gossip id, qualifier and object need to disseminate. + */ +final class Gossip { + + private final String gossipId; + private final Message message; + + public Gossip(String gossipId, Message message) { + checkArgument(gossipId != null); + checkArgument(message != null); + this.gossipId = gossipId; + this.message = message; + } + + public String gossipId() { + return gossipId; + } + + public Message message() { + return message; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + if (that == null || getClass() != that.getClass()) { + return false; + } + Gossip gossip = (Gossip) that; + return Objects.equals(gossipId, gossip.gossipId) + && Objects.equals(message, gossip.message); + } + + @Override + public int hashCode() { + return Objects.hash(gossipId, message); + } + + @Override + public String toString() { + return "Gossip{gossipId=" + gossipId + ", message=" + message + '}'; + } +} diff --git a/cluster/src/main/java/io/scalecube/cluster/gossip/GossipConfig.java b/cluster/src/main/java/io/scalecube/cluster/gossip/GossipConfig.java new file mode 100644 index 00000000..ec3351d9 --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/gossip/GossipConfig.java @@ -0,0 +1,11 @@ +package io.scalecube.cluster.gossip; + +public interface GossipConfig { + + int getGossipFanout(); + + long getGossipInterval(); + + int getGossipRepeatMult(); + +} diff --git a/cluster/src/main/java/io/scalecube/cluster/gossip/GossipProtocol.java b/cluster/src/main/java/io/scalecube/cluster/gossip/GossipProtocol.java new file mode 100644 index 00000000..1a7e66e9 --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/gossip/GossipProtocol.java @@ -0,0 +1,39 @@ +package io.scalecube.cluster.gossip; + +import io.scalecube.transport.Message; + +import rx.Observable; + +import java.util.concurrent.CompletableFuture; + +/** + * Gossip Protocol component responsible for spreading information (gossips) over the cluster members using + * infection-style dissemination algorithms. It provides reliable cross-cluster broadcast. + * + * @author Anton Kharenko + */ +public interface GossipProtocol { + + /** + * Starts running gossip protocol. After started it begins to receive and send gossip messages + */ + void start(); + + /** + * Stops running gossip protocol and releases occupied resources. + */ + void stop(); + + /** + * Spreads given message between cluster members. + * + * @return future result with gossip id once gossip fully spread. + */ + CompletableFuture spread(Message message); + + /** + * Listens for gossips from other cluster members. + */ + Observable listen(); + +} diff --git a/cluster/src/main/java/io/scalecube/cluster/gossip/GossipProtocolImpl.java b/cluster/src/main/java/io/scalecube/cluster/gossip/GossipProtocolImpl.java new file mode 100644 index 00000000..7edb3966 --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/gossip/GossipProtocolImpl.java @@ -0,0 +1,309 @@ +package io.scalecube.cluster.gossip; + +import static com.google.common.base.Preconditions.checkArgument; + +import io.scalecube.cluster.ClusterMath; +import io.scalecube.cluster.Member; +import io.scalecube.cluster.membership.MembershipProtocol; +import io.scalecube.cluster.membership.MembershipEvent; +import io.scalecube.transport.Transport; +import io.scalecube.transport.Message; + +import com.google.common.collect.Maps; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import rx.Observable; +import rx.Scheduler; +import rx.Subscriber; +import rx.observers.Subscribers; +import rx.schedulers.Schedulers; +import rx.subjects.PublishSubject; +import rx.subjects.Subject; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +public final class GossipProtocolImpl implements GossipProtocol { + + private static final Logger LOGGER = LoggerFactory.getLogger(GossipProtocolImpl.class); + + // Qualifiers + + public static final String GOSSIP_REQ = "sc/gossip/req"; + + // Injected + + private final Transport transport; + private final MembershipProtocol membership; + private final GossipConfig config; + + // Local State + + private long period = 0; + private long gossipCounter = 0; + private Map gossips = Maps.newHashMap(); + private Map> futures = Maps.newHashMap(); + + private List remoteMembers = new ArrayList<>(); + private int remoteMembersIndex = -1; + + // Subscriptions + + private Subscriber onMemberAddedEventSubscriber; + private Subscriber onMemberRemovedEventSubscriber; + private Subscriber onGossipRequestSubscriber; + + // Subject + + private Subject subject = PublishSubject.create().toSerialized(); + + // Scheduled + + private final ScheduledExecutorService executor; + private final Scheduler scheduler; + private ScheduledFuture spreadGossipTask; + + /** + * Creates new instance of gossip protocol with given memberId, transport and settings. + * + * @param transport transport + * @param membership membership protocol + * @param config gossip protocol settings + */ + public GossipProtocolImpl(Transport transport, MembershipProtocol membership, GossipConfig config) { + checkArgument(transport != null); + checkArgument(membership != null); + checkArgument(config != null); + this.transport = transport; + this.membership = membership; + this.config = config; + String nameFormat = "sc-gossip-" + Integer.toString(membership.member().address().port()); + this.executor = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat(nameFormat).setDaemon(true).build()); + this.scheduler = Schedulers.from(executor); + } + + /** + * NOTE: this method is for testing purpose only. + */ + Transport getTransport() { + return transport; + } + + /** + * NOTE: this method is for testing purpose only. + */ + Member getMember() { + return membership.member(); + } + + @Override + public void start() { + onMemberAddedEventSubscriber = Subscribers.create(remoteMembers::add, this::onError); + membership.listen().observeOn(scheduler) + .filter(MembershipEvent::isAdded) + .map(MembershipEvent::member) + .subscribe(onMemberAddedEventSubscriber); + + onMemberRemovedEventSubscriber = Subscribers.create(remoteMembers::remove, this::onError); + membership.listen().observeOn(scheduler) + .filter(MembershipEvent::isRemoved) + .map(MembershipEvent::member) + .subscribe(onMemberRemovedEventSubscriber); + + onGossipRequestSubscriber = Subscribers.create(this::onGossipReq, this::onError); + transport.listen().observeOn(scheduler) + .filter(this::isGossipReq) + .subscribe(onGossipRequestSubscriber); + + spreadGossipTask = executor.scheduleWithFixedDelay(this::doSpreadGossip, + config.getGossipInterval(), config.getGossipInterval(), TimeUnit.MILLISECONDS); + } + + private void onError(Throwable throwable) { + LOGGER.error("Received unexpected error: ", throwable); + } + + @Override + public void stop() { + // Stop accepting gossip requests + if (onMemberAddedEventSubscriber != null) { + onMemberAddedEventSubscriber.unsubscribe(); + } + if (onMemberRemovedEventSubscriber != null) { + onMemberRemovedEventSubscriber.unsubscribe(); + } + if (onGossipRequestSubscriber != null) { + onGossipRequestSubscriber.unsubscribe(); + } + + // Stop spreading gossips + if (spreadGossipTask != null) { + spreadGossipTask.cancel(true); + } + + // Shutdown executor + // TODO AK: Consider to await termination ?! + executor.shutdown(); + + // Stop publishing events + subject.onCompleted(); + } + + @Override + public CompletableFuture spread(Message message) { + CompletableFuture future = new CompletableFuture<>(); + executor.execute(() -> futures.put(onSpreadGossip(message), future)); + return future; + } + + @Override + public Observable listen() { + return subject.onBackpressureBuffer().asObservable(); + } + + // ================================================ + // ============== Action Methods ================== + // ================================================ + + private void doSpreadGossip() { + // Increment period + period++; + + // Check any gossips exists + if (gossips.isEmpty()) { + return; // nothing to spread + } + + try { + // Spread gossips to randomly selected member(s) + selectGossipMembers().forEach(this::spreadGossipsTo); + + // Sweep gossips + sweepGossips(); + } catch (Exception cause) { + LOGGER.error("Exception on sending GossipReq[{}] exception: {}", period, cause.getMessage(), cause); + } + } + + // ================================================ + // ============== Event Listeners ================= + // ================================================ + + private String onSpreadGossip(Message message) { + Gossip gossip = new Gossip(generateGossipId(), message); + GossipState gossipState = new GossipState(gossip, period); + gossips.put(gossip.gossipId(), gossipState); + return gossip.gossipId(); + } + + private void onGossipReq(Message message) { + GossipRequest gossipRequest = message.data(); + for (Gossip gossip : gossipRequest.gossips()) { + GossipState gossipState = gossips.get(gossip.gossipId()); + if (gossipState == null) { // new gossip + gossipState = new GossipState(gossip, period); + gossips.put(gossip.gossipId(), gossipState); + subject.onNext(gossip.message()); + } + gossipState.addToInfected(gossipRequest.from()); + } + } + + // ================================================ + // ============== Helper Methods ================== + // ================================================ + + private boolean isGossipReq(Message message) { + return GOSSIP_REQ.equals(message.qualifier()); + } + + private String generateGossipId() { + return membership.member().id() + "-" + gossipCounter++; + } + + private void spreadGossipsTo(Member member) { + // Select gossips to send + List gossipsToSend = selectGossipsToSend(member); + if (gossipsToSend.isEmpty()) { + return; // nothing to spread + } + + // Send gossip request + Message gossipReqMsg = buildGossipRequestMessage(gossipsToSend); + transport.send(member.address(), gossipReqMsg); + } + + private List selectGossipsToSend(Member member) { + int periodsToSpread = + ClusterMath.gossipPeriodsToSpread(config.getGossipRepeatMult(), remoteMembers.size() + 1); + return gossips.values().stream() + .filter(gossipState -> gossipState.infectionPeriod() + periodsToSpread >= period) // max rounds + .filter(gossipState -> !gossipState.isInfected(member.id())) // already infected + .map(GossipState::gossip) + .collect(Collectors.toList()); + } + + private List selectGossipMembers() { + int gossipFanout = config.getGossipFanout(); + if (remoteMembers.size() < gossipFanout) { // select all + return remoteMembers; + } else { // select random members + // Shuffle members initially and once reached top bound + if (remoteMembersIndex < 0 || remoteMembersIndex + gossipFanout > remoteMembers.size()) { + Collections.shuffle(remoteMembers); + remoteMembersIndex = 0; + } + + // Select members + List selectedMembers = gossipFanout == 1 + ? Collections.singletonList(remoteMembers.get(remoteMembersIndex)) + : remoteMembers.subList(remoteMembersIndex, remoteMembersIndex + gossipFanout); + + // Increment index and return result + remoteMembersIndex += gossipFanout; + return selectedMembers; + } + } + + private Message buildGossipRequestMessage(List gossipsToSend) { + GossipRequest gossipReqData = new GossipRequest(gossipsToSend, membership.member().id()); + return Message.withData(gossipReqData).qualifier(GOSSIP_REQ).build(); + } + + private void sweepGossips() { + // Select gossips to sweep + int periodsToSweep = ClusterMath.gossipPeriodsToSweep(config.getGossipRepeatMult(), remoteMembers.size() + 1); + Set gossipsToRemove = gossips.values().stream() + .filter(gossipState -> period > gossipState.infectionPeriod() + periodsToSweep) + .collect(Collectors.toSet()); + + // Check if anything selected + if (gossipsToRemove.isEmpty()) { + return; // nothing to sweep + } + + // Sweep gossips + LOGGER.debug("Sweep gossips: {}", gossipsToRemove); + for (GossipState gossipState : gossipsToRemove) { + gossips.remove(gossipState.gossip().gossipId()); + CompletableFuture future = futures.remove(gossipState.gossip().gossipId()); + if (future != null) { + future.complete(gossipState.gossip().gossipId()); + } + } + } + +} diff --git a/cluster/src/main/java/io/scalecube/cluster/gossip/GossipRequest.java b/cluster/src/main/java/io/scalecube/cluster/gossip/GossipRequest.java new file mode 100644 index 00000000..8bb3de6c --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/gossip/GossipRequest.java @@ -0,0 +1,35 @@ +package io.scalecube.cluster.gossip; + +import io.protostuff.Tag; + +import java.util.ArrayList; +import java.util.List; + +/** + * Gossip request which be transmitted through the network, contains list of gossips. + */ +final class GossipRequest { + @Tag(1) + private final List gossips; + + @Tag(2) + private final String from; + + public GossipRequest(List gossips, String from) { + this.gossips = new ArrayList<>(gossips); + this.from = from; + } + + public List gossips() { + return gossips; + } + + public String from() { + return from; + } + + @Override + public String toString() { + return "GossipRequest{gossips=" + gossips + ", from=" + from + '}'; + } +} diff --git a/cluster/src/main/java/io/scalecube/cluster/gossip/GossipState.java b/cluster/src/main/java/io/scalecube/cluster/gossip/GossipState.java new file mode 100644 index 00000000..5022feff --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/gossip/GossipState.java @@ -0,0 +1,49 @@ +package io.scalecube.cluster.gossip; + +import com.google.common.base.Preconditions; + +import java.util.HashSet; +import java.util.Set; + +/** Data related to gossip, maintained locally on each node. */ +final class GossipState { + + /** Target gossip. */ + private final Gossip gossip; + + /** Local gossip period when gossip was received for the first time. */ + private final long infectionPeriod; + + /** Set of member IDs this gossip was received from. */ + private final Set infected = new HashSet<>(); + + GossipState(Gossip gossip, long infectionPeriod) { + Preconditions.checkArgument(gossip != null); + this.gossip = gossip; + this.infectionPeriod = infectionPeriod; + } + + public Gossip gossip() { + return gossip; + } + + public long infectionPeriod() { + return infectionPeriod; + } + + public void addToInfected(String memberId) { + infected.add(memberId); + } + + public boolean isInfected(String memberId) { + return infected.contains(memberId); + } + + @Override + public String toString() { + return "GossipState{gossip=" + gossip + + ", infectionPeriod=" + infectionPeriod + + ", infected=" + infected + + '}'; + } +} diff --git a/cluster/src/main/java/io/scalecube/cluster/membership/MemberStatus.java b/cluster/src/main/java/io/scalecube/cluster/membership/MemberStatus.java new file mode 100644 index 00000000..48590adf --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/membership/MemberStatus.java @@ -0,0 +1,21 @@ +package io.scalecube.cluster.membership; + +public enum MemberStatus { + + /** + * Member is reachable and responding on messages. + */ + ALIVE, + + /** + * Member can't be reached and marked as suspected to be failed. + */ + SUSPECT, + + /** + * Member declared as dead after being {@link #SUSPECT} for configured time or when node has been gracefully shutdown + * and left cluster. + */ + DEAD + +} diff --git a/cluster/src/main/java/io/scalecube/cluster/membership/MembershipConfig.java b/cluster/src/main/java/io/scalecube/cluster/membership/MembershipConfig.java new file mode 100644 index 00000000..6f29b6d7 --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/membership/MembershipConfig.java @@ -0,0 +1,28 @@ +package io.scalecube.cluster.membership; + +import io.scalecube.transport.Address; + +import java.util.List; +import java.util.Map; + +public interface MembershipConfig { + + List
getSeedMembers(); + + Map getMetadata(); + + int getSyncInterval(); + + int getSyncTimeout(); + + String getSyncGroup(); + + int getPingInterval(); + + int getSuspicionMult(); + + String getMemberHost(); + + Integer getMemberPort(); + +} diff --git a/cluster/src/main/java/io/scalecube/cluster/membership/MembershipEvent.java b/cluster/src/main/java/io/scalecube/cluster/membership/MembershipEvent.java new file mode 100644 index 00000000..cf608812 --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/membership/MembershipEvent.java @@ -0,0 +1,79 @@ +package io.scalecube.cluster.membership; + +import static com.google.common.base.Preconditions.checkArgument; + +import io.scalecube.cluster.Member; + +import javax.annotation.concurrent.Immutable; + +/** + * Event which is emitted on cluster membership changes when new member added or removed from cluster. + * + * @author Anton Kharenko + */ +@Immutable +public final class MembershipEvent { + + public enum Type { + ADDED, REMOVED, UPDATED + } + + private final Type type; + private final Member newMember; + private final Member oldMember; + + private MembershipEvent(Type type, Member oldMember, Member newMember) { + checkArgument(type != null); + this.type = type; + this.oldMember = oldMember; + this.newMember = newMember; + } + + static MembershipEvent createRemoved(Member member) { + return new MembershipEvent(Type.REMOVED, member, null); + } + + static MembershipEvent createAdded(Member member) { + return new MembershipEvent(Type.ADDED, null, member); + } + + static MembershipEvent createUpdated(Member oldMember, Member newMember) { + return new MembershipEvent(Type.UPDATED, oldMember, newMember); + } + + public Type type() { + return type; + } + + public boolean isAdded() { + return type == Type.ADDED; + } + + public boolean isRemoved() { + return type == Type.REMOVED; + } + + public boolean isUpdated() { + return type == Type.UPDATED; + } + + public Member member() { + return type == Type.REMOVED ? oldMember : newMember; + } + + public Member oldMember() { + return oldMember; + } + + public Member newMember() { + return newMember; + } + + @Override + public String toString() { + return "MembershipEvent{type=" + type + + ", newMember=" + newMember + + ", oldMember=" + oldMember + + '}'; + } +} diff --git a/cluster/src/main/java/io/scalecube/cluster/membership/MembershipProtocol.java b/cluster/src/main/java/io/scalecube/cluster/membership/MembershipProtocol.java new file mode 100644 index 00000000..98d2818b --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/membership/MembershipProtocol.java @@ -0,0 +1,37 @@ +package io.scalecube.cluster.membership; + +import io.scalecube.cluster.Member; + +import rx.Observable; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +/** + * Cluster Membership Protocol component responsible for managing information about existing members of the cluster. + * + * @author Anton Kharenko + */ +public interface MembershipProtocol { + + /** + * Returns local cluster member. + */ + Member member(); + + /** + * Updates local member metadata. + */ + void updateMetadata(Map metadata); + + /** + * Updates local member metadata to set given key and value. + */ + void updateMetadataProperty(String key, String value); + + /** + * Listen changes in cluster membership. + */ + Observable listen(); + +} diff --git a/cluster/src/main/java/io/scalecube/cluster/membership/MembershipProtocolImpl.java b/cluster/src/main/java/io/scalecube/cluster/membership/MembershipProtocolImpl.java new file mode 100644 index 00000000..40ea02be --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/membership/MembershipProtocolImpl.java @@ -0,0 +1,561 @@ +package io.scalecube.cluster.membership; + +import static io.scalecube.cluster.membership.MemberStatus.ALIVE; +import static io.scalecube.cluster.membership.MemberStatus.DEAD; + +import io.scalecube.cluster.ClusterMath; +import io.scalecube.cluster.Member; +import io.scalecube.cluster.fdetector.FailureDetectorEvent; +import io.scalecube.cluster.fdetector.FailureDetector; +import io.scalecube.cluster.gossip.GossipProtocol; +import io.scalecube.transport.Address; +import io.scalecube.transport.Message; +import io.scalecube.transport.Transport; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import rx.Observable; +import rx.Scheduler; +import rx.Subscriber; +import rx.observers.Subscribers; +import rx.schedulers.Schedulers; +import rx.subjects.PublishSubject; +import rx.subjects.Subject; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +public final class MembershipProtocolImpl implements MembershipProtocol { + + private static final Logger LOGGER = LoggerFactory.getLogger(MembershipProtocolImpl.class); + + private enum MembershipUpdateReason { + FAILURE_DETECTOR_EVENT, + MEMBERSHIP_GOSSIP, + SYNC, + INITIAL_SYNC, + SUSPICION_TIMEOUT + } + + // Qualifiers + + public static final String SYNC = "sc/membership/sync"; + public static final String SYNC_ACK = "sc/membership/syncAck"; + public static final String MEMBERSHIP_GOSSIP = "sc/membership/gossip"; + + // Injected + + private final AtomicReference memberRef; + private final Transport transport; + private final MembershipConfig config; + private final List
seedMembers; + private FailureDetector failureDetector; + private GossipProtocol gossipProtocol; + + // State + + private final Map membershipTable = new HashMap<>(); + + // Subject + + private final Subject subject = + PublishSubject.create().toSerialized(); + + // Subscriptions + + private Subscriber onSyncRequestSubscriber; + private Subscriber onSyncAckResponseSubscriber; + private Subscriber onFdEventSubscriber; + private Subscriber onGossipRequestSubscriber; + + // Scheduled + + private final Scheduler scheduler; + private final ScheduledExecutorService executor; + private final Map> suspicionTimeoutTasks = new HashMap<>(); + private ScheduledFuture syncTask; + + /** + * Creates new instantiates of cluster membership protocol with given transport and config. + * + * @param transport transport + * @param config membership config parameters + */ + public MembershipProtocolImpl(Transport transport, MembershipConfig config) { + this.transport = transport; + this.config = config; + + Address address = memberAddress(transport, config); + Member member = new Member(IdGenerator.generateId(), address, config.getMetadata()); + this.memberRef = new AtomicReference<>(member); + + String nameFormat = "sc-membership-" + Integer.toString(address.port()); + this.executor = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat(nameFormat).setDaemon(true).build()); + + this.scheduler = Schedulers.from(executor); + this.seedMembers = cleanUpSeedMembers(config.getSeedMembers()); + } + + /** + * Returns the accessible member address, either from the transport or the overridden variables. + * @param transport transport + * @param config membership config parameters + * @return Accessible member address + */ + protected static Address memberAddress(Transport transport, MembershipConfig config) { + Address memberAddress = transport.address(); + if (config.getMemberHost() != null) { + int memberPort = config.getMemberPort() != null ? config.getMemberPort() : memberAddress.port(); + memberAddress = Address.create(config.getMemberHost(), memberPort); + } + + return memberAddress; + } + + // Remove duplicates and local address + private List
cleanUpSeedMembers(Collection
seedMembers) { + Set
seedMembersSet = new HashSet<>(seedMembers); // remove duplicates + seedMembersSet.remove(member().address()); // remove local address + return Collections.unmodifiableList(new ArrayList<>(seedMembersSet)); + } + + public void setFailureDetector(FailureDetector failureDetector) { + this.failureDetector = failureDetector; + } + + public void setGossipProtocol(GossipProtocol gossipProtocol) { + this.gossipProtocol = gossipProtocol; + } + + /** + * NOTE: this method is for testing purpose only. + */ + FailureDetector getFailureDetector() { + return failureDetector; + } + + /** + * NOTE: this method is for testing purpose only. + */ + GossipProtocol getGossipProtocol() { + return gossipProtocol; + } + + /** + * NOTE: this method is for testing purpose only. + */ + Transport getTransport() { + return transport; + } + + /** + * NOTE: this method is for testing purpose only. + */ + List getMembershipRecords() { + return ImmutableList.copyOf(membershipTable.values()); + } + + @Override + public Observable listen() { + return subject.onBackpressureBuffer().asObservable(); + } + + @Override + public Member member() { + return memberRef.get(); + } + + @Override + public void updateMetadata(Map metadata) { + executor.execute(() -> onUpdateMetadata(metadata)); + } + + @Override + public void updateMetadataProperty(String key, String value) { + executor.execute(() -> onUpdateMetadataProperty(key, value)); + } + + /** + * Spreads leave notification to other cluster members. + */ + public CompletableFuture leave() { + CompletableFuture future = new CompletableFuture<>(); + executor.execute(() -> { + CompletableFuture leaveFuture = onLeave(); + leaveFuture.whenComplete((gossipId, error) -> { + future.complete(gossipId); + }); + }); + return future; + } + + /** + * Starts running cluster membership protocol. After started it begins to receive and send cluster membership messages + */ + public CompletableFuture start() { + // Init membership table with local member record + Member member = memberRef.get(); + MembershipRecord localMemberRecord = new MembershipRecord(member, ALIVE, 0); + membershipTable.put(member.id(), localMemberRecord); + + // Listen to incoming SYNC requests from other members + onSyncRequestSubscriber = Subscribers.create(this::onSync, this::onError); + transport.listen().observeOn(scheduler) + .filter(msg -> SYNC.equals(msg.qualifier())) + .filter(this::checkSyncGroup) + .subscribe(onSyncRequestSubscriber); + + // Listen to incoming SYNC ACK responses from other members + onSyncAckResponseSubscriber = Subscribers.create(this::onSyncAck, this::onError); + transport.listen().observeOn(scheduler) + .filter(msg -> SYNC_ACK.equals(msg.qualifier())) + .filter(msg -> msg.correlationId() == null) // filter out initial sync + .filter(this::checkSyncGroup) + .subscribe(onSyncAckResponseSubscriber); + + // Listen to events from failure detector + onFdEventSubscriber = Subscribers.create(this::onFailureDetectorEvent, this::onError); + failureDetector.listen().observeOn(scheduler) + .subscribe(onFdEventSubscriber); + + // Listen to membership gossips + onGossipRequestSubscriber = Subscribers.create(this::onMembershipGossip, this::onError); + gossipProtocol.listen().observeOn(scheduler) + .filter(msg -> MEMBERSHIP_GOSSIP.equals(msg.qualifier())) + .subscribe(onGossipRequestSubscriber); + + // Make initial sync with all seed members + return doInitialSync(); + } + + private void onError(Throwable throwable) { + LOGGER.error("Received unexpected error: ", throwable); + } + + /** + * Stops running cluster membership protocol and releases occupied resources. + */ + public void stop() { + // Stop accepting requests and events + if (onSyncRequestSubscriber != null) { + onSyncRequestSubscriber.unsubscribe(); + } + if (onFdEventSubscriber != null) { + onFdEventSubscriber.unsubscribe(); + } + if (onGossipRequestSubscriber != null) { + onGossipRequestSubscriber.unsubscribe(); + } + if (onSyncAckResponseSubscriber != null) { + onSyncAckResponseSubscriber.unsubscribe(); + } + + // Stop sending sync + if (syncTask != null) { + syncTask.cancel(true); + } + + // Cancel remove members tasks + for (String memberId : suspicionTimeoutTasks.keySet()) { + ScheduledFuture future = suspicionTimeoutTasks.get(memberId); + if (future != null) { + future.cancel(true); + } + } + suspicionTimeoutTasks.clear(); + + // Shutdown executor + executor.shutdown(); + + // Stop publishing events + subject.onCompleted(); + } + + // ================================================ + // ============== Action Methods ================== + // ================================================ + + private CompletableFuture doInitialSync() { + LOGGER.debug("Making initial Sync to all seed members: {}", seedMembers); + if (seedMembers.isEmpty()) { + schedulePeriodicSync(); + return CompletableFuture.completedFuture(null); + } + + CompletableFuture syncResponseFuture = new CompletableFuture<>(); + + // Listen initial Sync Ack + String cid = memberRef.get().id(); + transport.listen().observeOn(scheduler) + .filter(msg -> SYNC_ACK.equals(msg.qualifier())) + .filter(msg -> cid.equals(msg.correlationId())) + .filter(this::checkSyncGroup) + .take(1) + .timeout(config.getSyncTimeout(), TimeUnit.MILLISECONDS, scheduler) + .subscribe( + message -> { + SyncData syncData = message.data(); + LOGGER.info("Joined cluster '{}': {}", syncData.getSyncGroup(), syncData.getMembership()); + onSyncAck(message, true); + schedulePeriodicSync(); + syncResponseFuture.complete(null); + }, + throwable -> { + LOGGER.info("Timeout getting initial SyncAck from seed members: {}", seedMembers); + schedulePeriodicSync(); + syncResponseFuture.complete(null); + }); + + Message syncMsg = prepareSyncDataMsg(SYNC, cid); + seedMembers.forEach(address -> transport.send(address, syncMsg)); + + return syncResponseFuture; + } + + private void doSync() { + try { + Address syncMember = selectSyncAddress(); + if (syncMember == null) { + return; + } + Message syncMsg = prepareSyncDataMsg(SYNC, null); + transport.send(syncMember, syncMsg); + LOGGER.debug("Send Sync to {}: {}", syncMember, syncMsg); + } catch (Exception cause) { + LOGGER.error("Unhandled exception: {}", cause, cause); + } + } + + // ================================================ + // ============== Event Listeners ================= + // ================================================ + + private void onUpdateMetadataProperty(String key, String value) { + // Update local member reference + Member curMember = memberRef.get(); + Map metadata = new HashMap<>(curMember.metadata()); + metadata.put(key, value); + onUpdateMetadata(metadata); + } + + private void onUpdateMetadata(Map metadata) { + // Update local member reference + Member curMember = memberRef.get(); + String memberId = curMember.id(); + Member newMember = new Member(memberId, curMember.address(), metadata); + memberRef.set(newMember); + + // Update membership table + MembershipRecord curRecord = membershipTable.get(memberId); + MembershipRecord newRecord = new MembershipRecord(newMember, ALIVE, curRecord.incarnation() + 1); + membershipTable.put(memberId, newRecord); + + // Emit membership updated event + subject.onNext(MembershipEvent.createUpdated(curMember, newMember)); + + // Spread new membership record over the cluster + spreadMembershipGossip(newRecord); + } + + private void onSyncAck(Message syncAckMsg) { + onSyncAck(syncAckMsg, false); + } + + private void onSyncAck(Message syncAckMsg, boolean initial) { + LOGGER.debug("Received SyncAck: {}", syncAckMsg); + syncMembership(syncAckMsg.data(), initial); + } + + /** + * Merges incoming SYNC data, merges it and sending back merged data with SYNC_ACK. + */ + private void onSync(Message syncMsg) { + LOGGER.debug("Received Sync: {}", syncMsg); + syncMembership(syncMsg.data(), false); + Message syncAckMsg = prepareSyncDataMsg(SYNC_ACK, syncMsg.correlationId()); + transport.send(syncMsg.sender(), syncAckMsg); + } + + /** + * Merges FD updates and processes them. + */ + private void onFailureDetectorEvent(FailureDetectorEvent fdEvent) { + MembershipRecord r0 = membershipTable.get(fdEvent.member().id()); + if (r0 == null) { // member already removed + return; + } + if (r0.status() == fdEvent.status()) { // status not changed + return; + } + LOGGER.debug("Received status change on failure detector event: {}", fdEvent); + if (fdEvent.status() == ALIVE) { + // TODO: Consider to make more elegant solution + // Alive won't override SUSPECT so issue instead extra sync with member to force it spread alive with inc + 1 + Message syncMsg = prepareSyncDataMsg(SYNC, null); + transport.send(fdEvent.member().address(), syncMsg); + } else { + MembershipRecord r1 = new MembershipRecord(r0.member(), fdEvent.status(), r0.incarnation()); + updateMembership(r1, MembershipUpdateReason.FAILURE_DETECTOR_EVENT); + } + } + + /** + * Merges received membership gossip (not spreading gossip further). + */ + private void onMembershipGossip(Message message) { + MembershipRecord record = message.data(); + LOGGER.debug("Received membership gossip: {}", record); + updateMembership(record, MembershipUpdateReason.MEMBERSHIP_GOSSIP); + } + + // ================================================ + // ============== Helper Methods ================== + // ================================================ + + private Address selectSyncAddress() { + // TODO [AK]: During running phase it should send to both seed or not seed members (issue #38) + return !seedMembers.isEmpty() ? seedMembers.get(ThreadLocalRandom.current().nextInt(seedMembers.size())) : null; + } + + private boolean checkSyncGroup(Message message) { + SyncData data = message.data(); + return config.getSyncGroup().equals(data.getSyncGroup()); + } + + private void schedulePeriodicSync() { + int syncInterval = config.getSyncInterval(); + syncTask = executor.scheduleWithFixedDelay(this::doSync, syncInterval, syncInterval, TimeUnit.MILLISECONDS); + } + + private Message prepareSyncDataMsg(String qualifier, String cid) { + List membershipRecords = new ArrayList<>(membershipTable.values()); + SyncData syncData = new SyncData(membershipRecords, config.getSyncGroup()); + return Message.withData(syncData).qualifier(qualifier).correlationId(cid).build(); + } + + private void syncMembership(SyncData syncData, boolean initial) { + for (MembershipRecord r1 : syncData.getMembership()) { + MembershipRecord r0 = membershipTable.get(r1.id()); + if (!r1.equals(r0)) { + MembershipUpdateReason reason = initial ? MembershipUpdateReason.INITIAL_SYNC : MembershipUpdateReason.SYNC; + updateMembership(r1, reason); + } + } + } + + /** + * Try to update membership table with the given record. + * + * @param r1 new membership record which compares with existing r0 record + * @param reason indicating the reason for updating membership table + */ + private void updateMembership(MembershipRecord r1, MembershipUpdateReason reason) { + Preconditions.checkArgument(r1 != null, "Membership record can't be null"); + + // Get current record + MembershipRecord r0 = membershipTable.get(r1.id()); + + // Check if new record r1 overrides existing membership record r0 + if (!r1.isOverrides(r0)) { + return; + } + + // If received updated for local member then increase incarnation number and spread Alive gossip + Member localMember = memberRef.get(); + if (r1.member().id().equals(localMember.id())) { + int currentIncarnation = Math.max(r0.incarnation(), r1.incarnation()); + MembershipRecord r2 = new MembershipRecord(localMember, r0.status(), currentIncarnation + 1); + membershipTable.put(localMember.id(), r2); + LOGGER.debug("Local membership record r0={}, but received r1={}, spread r2={}", r0, r1, r2); + spreadMembershipGossip(r2); + return; + } + + // Update membership + if (r1.isDead()) { + membershipTable.remove(r1.id()); + } else { + membershipTable.put(r1.id(), r1); + } + + // Schedule/cancel suspicion timeout task + if (r1.isSuspect()) { + scheduleSuspicionTimeoutTask(r1); + } else { + cancelSuspicionTimeoutTask(r1.id()); + } + + // Emit membership event + if (r1.isDead()) { + subject.onNext(MembershipEvent.createRemoved(r1.member())); + } else if (r0 == null && r1.isAlive()) { + subject.onNext(MembershipEvent.createAdded(r1.member())); + } else if (r0 != null && !r0.member().equals(r1.member())) { + subject.onNext(MembershipEvent.createUpdated(r0.member(), r1.member())); + } + + + // Spread gossip (unless already gossiped) + if (reason != MembershipUpdateReason.MEMBERSHIP_GOSSIP && reason != MembershipUpdateReason.INITIAL_SYNC) { + spreadMembershipGossip(r1); + } + } + + private void cancelSuspicionTimeoutTask(String memberId) { + ScheduledFuture future = suspicionTimeoutTasks.remove(memberId); + if (future != null) { + future.cancel(true); + } + } + + private void scheduleSuspicionTimeoutTask(MembershipRecord record) { + long suspicionTimeout = + ClusterMath.suspicionTimeout(config.getSuspicionMult(), membershipTable.size(), config.getPingInterval()); + suspicionTimeoutTasks.computeIfAbsent(record.id(), + id -> executor.schedule(() -> onSuspicionTimeout(id), suspicionTimeout, TimeUnit.MILLISECONDS)); + } + + private void onSuspicionTimeout(String memberId) { + suspicionTimeoutTasks.remove(memberId); + MembershipRecord record = membershipTable.get(memberId); + if (record != null) { + LOGGER.debug("Declare SUSPECTED member as DEAD by timeout: {}", record); + MembershipRecord deadRecord = new MembershipRecord(record.member(), DEAD, record.incarnation()); + updateMembership(deadRecord, MembershipUpdateReason.SUSPICION_TIMEOUT); + } + } + + private CompletableFuture onLeave() { + Member curMember = memberRef.get(); + String memberId = curMember.id(); + MembershipRecord curRecord = membershipTable.get(memberId); + MembershipRecord newRecord = new MembershipRecord(this.member(), DEAD, curRecord.incarnation() + 1); + membershipTable.put(memberId, newRecord); + return spreadMembershipGossip(newRecord); + } + + private CompletableFuture spreadMembershipGossip(MembershipRecord record) { + Message membershipMsg = Message.withData(record).qualifier(MEMBERSHIP_GOSSIP).build(); + return gossipProtocol.spread(membershipMsg); + } + +} diff --git a/cluster/src/main/java/io/scalecube/cluster/membership/MembershipRecord.java b/cluster/src/main/java/io/scalecube/cluster/membership/MembershipRecord.java new file mode 100644 index 00000000..bf11ae4f --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/membership/MembershipRecord.java @@ -0,0 +1,120 @@ +package io.scalecube.cluster.membership; + +import static com.google.common.base.Preconditions.checkArgument; +import static io.scalecube.cluster.membership.MemberStatus.ALIVE; +import static io.scalecube.cluster.membership.MemberStatus.DEAD; +import static io.scalecube.cluster.membership.MemberStatus.SUSPECT; + +import io.scalecube.cluster.Member; +import io.scalecube.transport.Address; + +import java.util.Objects; + +import javax.annotation.Nonnull; +import javax.annotation.concurrent.Immutable; + +/** + * Cluster membership record which represents member, status, and incarnation. + */ +@Immutable +final class MembershipRecord { + + private final Member member; + private final MemberStatus status; + private final int incarnation; + + /** + * Instantiates new instance of membership record with given member, status and incarnation. + */ + public MembershipRecord(Member member, MemberStatus status, int incarnation) { + checkArgument(member != null); + checkArgument(status != null); + this.member = member; + this.status = status; + this.incarnation = incarnation; + } + + @Nonnull + public Member member() { + return member; + } + + @Nonnull + public String id() { + return member.id(); + } + + @Nonnull + public Address address() { + return member.address(); + } + + @Nonnull + public MemberStatus status() { + return status; + } + + public boolean isAlive() { + return status == ALIVE; + } + + public boolean isSuspect() { + return status == SUSPECT; + } + + public boolean isDead() { + return status == DEAD; + } + + public int incarnation() { + return incarnation; + } + + /** + * Checks either this record overrides given record. + * + * @param r0 existing record in membership table + * @return true if this record overrides exiting; false otherwise + */ + public boolean isOverrides(MembershipRecord r0) { + if (r0 == null) { + return isAlive(); + } + checkArgument(this.member.id().equals(r0.member.id()), "Can't compare records for different members"); + if (r0.status == DEAD) { + return false; + } + if (status == DEAD) { + return true; + } + if (incarnation == r0.incarnation) { + return (status != r0.status) && (status == SUSPECT); + } else { + return incarnation > r0.incarnation; + } + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + if (that == null || getClass() != that.getClass()) { + return false; + } + MembershipRecord record = (MembershipRecord) that; + return incarnation == record.incarnation + && Objects.equals(member, record.member) + && status == record.status; + } + + @Override + public int hashCode() { + return Objects.hash(member, status, incarnation); + } + + @Override + public String toString() { + return "{m: " + member + ", s: " + status + ", inc: " + incarnation + '}'; + } +} diff --git a/cluster/src/main/java/io/scalecube/cluster/membership/SyncData.java b/cluster/src/main/java/io/scalecube/cluster/membership/SyncData.java new file mode 100644 index 00000000..078d91cb --- /dev/null +++ b/cluster/src/main/java/io/scalecube/cluster/membership/SyncData.java @@ -0,0 +1,49 @@ +package io.scalecube.cluster.membership; + +import io.protostuff.Tag; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import javax.annotation.concurrent.Immutable; + +/** + * A class containing full membership table from specific member and used full synchronization between cluster members. + * + * @author Anton Kharenko + */ +@Immutable +final class SyncData { + + /** + * Full cluster membership table. + */ + @Tag(1) + private final List membership; + + /** + * Sort of cluster identifier. Only members in the same sync group allowed to join into cluster. + */ + @Tag(2) + private final String syncGroup; + + public SyncData(Collection membership, String syncGroup) { + this.membership = new ArrayList<>(membership); + this.syncGroup = syncGroup; + } + + public Collection getMembership() { + return new ArrayList<>(membership); + } + + public String getSyncGroup() { + return syncGroup; + } + + @Override + public String toString() { + return "SyncData{membership=" + membership + ", syncGroup=" + syncGroup + '}'; + } + +} diff --git a/cluster/src/test/java/io/scalecube/cluster/ClusterTest.java b/cluster/src/test/java/io/scalecube/cluster/ClusterTest.java new file mode 100644 index 00000000..7a111f65 --- /dev/null +++ b/cluster/src/test/java/io/scalecube/cluster/ClusterTest.java @@ -0,0 +1,207 @@ +package io.scalecube.cluster; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import io.scalecube.cluster.membership.MembershipEvent; +import io.scalecube.testlib.BaseTest; + +import com.google.common.collect.ImmutableMap; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class ClusterTest extends BaseTest { + + @Test + public void testJoinDynamicPort() throws Exception { + // Start seed node + Cluster seedNode = Cluster.joinAwait(); + + int membersNum = 10; + List otherNodes = new ArrayList<>(membersNum); + try { + // Start other nodes + long startAt = System.currentTimeMillis(); + for (int i = 0; i < membersNum; i++) { + otherNodes.add(Cluster.joinAwait(seedNode.address())); + } + LOGGER.info("Start up time: {} ms", System.currentTimeMillis() - startAt); + assertEquals(membersNum + 1, seedNode.members().size()); + LOGGER.info("Cluster nodes: {}", seedNode.members()); + } finally { + // Shutdown all nodes + shutdown(seedNode); + shutdown(otherNodes); + } + } + + @Test + public void testUpdateMetadata() throws Exception { + // Start seed member + Cluster seedNode = Cluster.joinAwait(); + + Cluster metadataNode = null; + int testMembersNum = 10; + List otherNodes = new ArrayList<>(testMembersNum); + try { + // Start member with metadata + Map metadata = ImmutableMap.of("key1", "value1", "key2", "value2"); + metadataNode = Cluster.joinAwait(metadata, seedNode.address()); + + // Start other test members + for (int i = 0; i < testMembersNum; i++) { + otherNodes.add(Cluster.joinAwait(seedNode.address())); + } + + // Check all test members know valid metadata + for (Cluster node : otherNodes) { + Optional mNodeOpt = node.member(metadataNode.member().id()); + assertTrue(mNodeOpt.isPresent()); + Member mNode = mNodeOpt.get(); + assertEquals(metadata, mNode.metadata()); + } + + // Subscribe for membership update event all nodes + CountDownLatch updateLatch = new CountDownLatch(testMembersNum); + for (Cluster node : otherNodes) { + node.listenMembership() + .filter(MembershipEvent::isUpdated) + .subscribe(event -> { + LOGGER.info("Received membership update event: {}", event); + updateLatch.countDown(); + }); + } + + // Update metadata + Map updatedMetadata = ImmutableMap.of("key1", "value3"); + metadataNode.updateMetadata(updatedMetadata); + + // Await latch + updateLatch.await(10, TimeUnit.SECONDS); + + // Check all nodes had updated metadata member + for (Cluster node : otherNodes) { + Optional mNodeOpt = node.member(metadataNode.member().id()); + assertTrue(mNodeOpt.isPresent()); + Member mNode = mNodeOpt.get(); + assertEquals(updatedMetadata, mNode.metadata()); + } + } finally { + // Shutdown all nodes + shutdown(seedNode); + shutdown(metadataNode); + shutdown(otherNodes); + } + } + + @Test + public void testUpdateMetadataProperty() throws Exception { + // Start seed member + Cluster seedNode = Cluster.joinAwait(); + + Cluster metadataNode = null; + int testMembersNum = 10; + List otherNodes = new ArrayList<>(testMembersNum); + + try { + // Start member with metadata + Map metadata = ImmutableMap.of("key1", "value1", "key2", "value2"); + metadataNode = Cluster.joinAwait(metadata, seedNode.address()); + + // Start other test members + for (int i = 0; i < testMembersNum; i++) { + otherNodes.add(Cluster.joinAwait(seedNode.address())); + } + + // Check all test members know valid metadata + for (Cluster node : otherNodes) { + Optional mNodeOpt = node.member(metadataNode.member().id()); + assertTrue(mNodeOpt.isPresent()); + Member mNode = mNodeOpt.get(); + assertEquals(metadata, mNode.metadata()); + } + + // Subscribe for membership update event all nodes + CountDownLatch updateLatch = new CountDownLatch(testMembersNum); + for (Cluster node : otherNodes) { + node.listenMembership() + .filter(MembershipEvent::isUpdated) + .subscribe(event -> { + LOGGER.info("Received membership update event: {}", event); + updateLatch.countDown(); + }); + } + + // Update metadata + metadataNode.updateMetadataProperty("key2", "value3"); + + // Await latch + updateLatch.await(10, TimeUnit.SECONDS); + + // Check all nodes had updated metadata member + for (Cluster node : otherNodes) { + Optional mNodeOpt = node.member(metadataNode.member().id()); + assertTrue(mNodeOpt.isPresent()); + Member mNode = mNodeOpt.get(); + Map mNodeMetadata = mNode.metadata(); + assertEquals(2, mNode.metadata().size()); + assertEquals("value1", mNodeMetadata.get("key1")); + assertEquals("value3", mNodeMetadata.get("key2")); + } + } finally { + // Shutdown all nodes + shutdown(seedNode); + shutdown(metadataNode); + shutdown(otherNodes); + } + } + + @Test + public void testShutdownCluster() throws Exception { + // Start seed member + Cluster seedNode = Cluster.joinAwait(); + + // Start nodes + Cluster node1 = Cluster.joinAwait(seedNode.address()); + Cluster node2 = Cluster.joinAwait(seedNode.address()); + Cluster node3 = Cluster.joinAwait(seedNode.address()); + + CountDownLatch leave = new CountDownLatch(1); + node2.shutdown().whenComplete((done, error) -> { + leave.countDown(); + }); + + leave.await(5, TimeUnit.SECONDS); + assertTrue(!seedNode.members().contains(node2.member())); + assertTrue(!node1.members().contains(node2.member())); + assertTrue(!node3.members().contains(node2.member())); + assertTrue(node2.isShutdown()); + + seedNode.shutdown(); + node1.shutdown(); + node3.shutdown(); + } + + private void shutdown(Cluster... nodes) { + shutdown(Arrays.asList(nodes)); + } + + private void shutdown(Collection nodes) { + for (Cluster node : nodes) { + try { + node.shutdown().get(); + } catch (Exception ex) { + LOGGER.error("Exception on cluster shutdown", ex); + } + } + } +} diff --git a/cluster/src/test/java/io/scalecube/cluster/fdetector/FailureDetectorTest.java b/cluster/src/test/java/io/scalecube/cluster/fdetector/FailureDetectorTest.java new file mode 100644 index 00000000..fd748e64 --- /dev/null +++ b/cluster/src/test/java/io/scalecube/cluster/fdetector/FailureDetectorTest.java @@ -0,0 +1,472 @@ +package io.scalecube.cluster.fdetector; + +import static com.google.common.base.Preconditions.checkArgument; +import static io.scalecube.cluster.membership.MemberStatus.SUSPECT; +import static io.scalecube.cluster.membership.MemberStatus.ALIVE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import io.scalecube.cluster.ClusterConfig; +import io.scalecube.cluster.Member; +import io.scalecube.cluster.membership.DummyMembershipProtocol; +import io.scalecube.cluster.membership.MembershipProtocol; +import io.scalecube.cluster.membership.MemberStatus; +import io.scalecube.testlib.BaseTest; +import io.scalecube.transport.Address; +import io.scalecube.transport.Transport; +import io.scalecube.transport.TransportConfig; + +import com.google.common.collect.Lists; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +public class FailureDetectorTest extends BaseTest { + + @Test + public void testTrusted() { + // Create transports + Transport a = Transport.bindAwait(true); + Transport b = Transport.bindAwait(true); + Transport c = Transport.bindAwait(true); + List
members = Arrays.asList(a.address(), b.address(), c.address()); + + // Create failure detectors + FailureDetectorImpl fd_a = createFD(a, members); + FailureDetectorImpl fd_b = createFD(b, members); + FailureDetectorImpl fd_c = createFD(c, members); + List fdetectors = Arrays.asList(fd_a, fd_b, fd_c); + + try { + start(fdetectors); + + Future> list_a = listenNextEventFor(fd_a, members); + Future> list_b = listenNextEventFor(fd_b, members); + Future> list_c = listenNextEventFor(fd_c, members); + + assertStatus(a.address(), ALIVE, awaitEvents(list_a), b.address(), c.address()); + assertStatus(b.address(), ALIVE, awaitEvents(list_b), a.address(), c.address()); + assertStatus(c.address(), ALIVE, awaitEvents(list_c), a.address(), b.address()); + } finally { + stop(fdetectors); + } + } + + @Test + public void testSuspected() { + // Create transports + Transport a = Transport.bindAwait(true); + Transport b = Transport.bindAwait(true); + Transport c = Transport.bindAwait(true); + List
members = Arrays.asList(a.address(), b.address(), c.address()); + + // Create failure detectors + FailureDetectorImpl fd_a = createFD(a, members); + FailureDetectorImpl fd_b = createFD(b, members); + FailureDetectorImpl fd_c = createFD(c, members); + List fdetectors = Arrays.asList(fd_a, fd_b, fd_c); + + // block all traffic + a.networkEmulator().block(members); + b.networkEmulator().block(members); + c.networkEmulator().block(members); + + try { + start(fdetectors); + + Future> list_a = listenNextEventFor(fd_a, members); + Future> list_b = listenNextEventFor(fd_b, members); + Future> list_c = listenNextEventFor(fd_c, members); + + assertStatus(a.address(), SUSPECT, awaitEvents(list_a), b.address(), c.address()); + assertStatus(b.address(), SUSPECT, awaitEvents(list_b), a.address(), c.address()); + assertStatus(c.address(), SUSPECT, awaitEvents(list_c), a.address(), b.address()); + } finally { + a.networkEmulator().unblockAll(); + b.networkEmulator().unblockAll(); + c.networkEmulator().unblockAll(); + stop(fdetectors); + } + } + + @Test + public void testTrustedDespiteBadNetwork() throws Exception { + // Create transports + Transport a = Transport.bindAwait(true); + Transport b = Transport.bindAwait(true); + Transport c = Transport.bindAwait(true); + List
members = Arrays.asList(a.address(), b.address(), c.address()); + + // Create failure detectors + FailureDetectorImpl fd_a = createFD(a, members); + FailureDetectorImpl fd_b = createFD(b, members); + FailureDetectorImpl fd_c = createFD(c, members); + List fdetectors = Arrays.asList(fd_a, fd_b, fd_c); + + // Traffic issue at connection A -> B + a.networkEmulator().block(b.address()); + + Future> list_a = listenNextEventFor(fd_a, members); + Future> list_b = listenNextEventFor(fd_b, members); + Future> list_c = listenNextEventFor(fd_c, members); + + try { + start(fdetectors); + + assertStatus(a.address(), ALIVE, awaitEvents(list_a), b.address(), c.address()); + assertStatus(b.address(), ALIVE, awaitEvents(list_b), a.address(), c.address()); + assertStatus(c.address(), ALIVE, awaitEvents(list_c), a.address(), b.address()); + } finally { + stop(fdetectors); + } + } + + @Test + public void testTrustedDespiteDifferentPingTimings() throws Exception { + // Create transports + Transport a = Transport.bindAwait(true); + Transport b = Transport.bindAwait(true); + Transport c = Transport.bindAwait(true); + List
members = Arrays.asList(a.address(), b.address(), c.address()); + + // Create failure detectors + FailureDetectorImpl fd_a = createFD(a, members); + FailureDetectorConfig fd_b_config = ClusterConfig.builder().pingTimeout(500).pingInterval(1000).build(); + FailureDetectorImpl fd_b = createFD(b, members, fd_b_config); + FailureDetectorImpl fd_c = createFD(c, members, ClusterConfig.defaultConfig()); + List fdetectors = Arrays.asList(fd_a, fd_b, fd_c); + + try { + start(fdetectors); + + Future> list_a = listenNextEventFor(fd_a, members); + Future> list_b = listenNextEventFor(fd_b, members); + Future> list_c = listenNextEventFor(fd_c, members); + + assertStatus(a.address(), ALIVE, awaitEvents(list_a), b.address(), c.address()); + assertStatus(b.address(), ALIVE, awaitEvents(list_b), a.address(), c.address()); + assertStatus(c.address(), ALIVE, awaitEvents(list_c), a.address(), b.address()); + } finally { + stop(fdetectors); + } + } + + @Test + public void testSuspectedMemberWithBadNetworkGetsPartitioned() throws Exception { + // Create transports + Transport a = Transport.bindAwait(true); + Transport b = Transport.bindAwait(true); + Transport c = Transport.bindAwait(true); + Transport d = Transport.bindAwait(true); + List
members = Arrays.asList(a.address(), b.address(), c.address(), d.address()); + + // Create failure detectors + FailureDetectorImpl fd_a = createFD(a, members); + FailureDetectorImpl fd_b = createFD(b, members); + FailureDetectorImpl fd_c = createFD(c, members); + FailureDetectorImpl fd_d = createFD(d, members); + List fdetectors = Arrays.asList(fd_a, fd_b, fd_c, fd_d); + + // Block traffic on member A to all cluster members + a.networkEmulator().block(members); + + try { + Future> list_a = listenNextEventFor(fd_a, members); + Future> list_b = listenNextEventFor(fd_b, members); + Future> list_c = listenNextEventFor(fd_c, members); + Future> list_d = listenNextEventFor(fd_d, members); + + start(fdetectors); + + assertStatus(a.address(), SUSPECT, awaitEvents(list_a), b.address(), c.address(), d.address()); // node A + // partitioned + assertStatus(b.address(), SUSPECT, awaitEvents(list_b), a.address()); + assertStatus(c.address(), SUSPECT, awaitEvents(list_c), a.address()); + assertStatus(d.address(), SUSPECT, awaitEvents(list_d), a.address()); + + // Unblock traffic on member A + a.networkEmulator().unblockAll(); + TimeUnit.SECONDS.sleep(4); + + list_a = listenNextEventFor(fd_a, members); + list_b = listenNextEventFor(fd_b, members); + list_c = listenNextEventFor(fd_c, members); + list_d = listenNextEventFor(fd_d, members); + + // Check member A recovers + + assertStatus(a.address(), ALIVE, awaitEvents(list_a), b.address(), c.address(), d.address()); + assertStatus(b.address(), ALIVE, awaitEvents(list_b), a.address(), c.address(), d.address()); + assertStatus(c.address(), ALIVE, awaitEvents(list_c), a.address(), b.address(), d.address()); + assertStatus(d.address(), ALIVE, awaitEvents(list_d), a.address(), b.address(), c.address()); + } finally { + stop(fdetectors); + } + } + + @Test + public void testSuspectedMemberWithNormalNetworkGetsPartitioned() throws Exception { + // Create transports + Transport a = Transport.bindAwait(true); + Transport b = Transport.bindAwait(true); + Transport c = Transport.bindAwait(true); + Transport d = Transport.bindAwait(true); + List
members = Arrays.asList(a.address(), b.address(), c.address(), d.address()); + + // Create failure detectors + FailureDetectorImpl fd_a = createFD(a, members); + FailureDetectorImpl fd_b = createFD(b, members); + FailureDetectorImpl fd_c = createFD(c, members); + FailureDetectorImpl fd_d = createFD(d, members); + List fdetectors = Arrays.asList(fd_a, fd_b, fd_c, fd_d); + + // Block traffic to node D on other members + a.networkEmulator().block(d.address()); + b.networkEmulator().block(d.address()); + c.networkEmulator().block(d.address()); + + try { + Future> list_a = listenNextEventFor(fd_a, members); + Future> list_b = listenNextEventFor(fd_b, members); + Future> list_c = listenNextEventFor(fd_c, members); + Future> list_d = listenNextEventFor(fd_d, members); + + start(fdetectors); + + assertStatus(a.address(), SUSPECT, awaitEvents(list_a), d.address()); + assertStatus(b.address(), SUSPECT, awaitEvents(list_b), d.address()); + assertStatus(c.address(), SUSPECT, awaitEvents(list_c), d.address()); + assertStatus(d.address(), SUSPECT, awaitEvents(list_d), a.address(), b.address(), c.address()); // node D + // partitioned + + // Unblock traffic to member D on other members + a.networkEmulator().unblockAll(); + b.networkEmulator().unblockAll(); + c.networkEmulator().unblockAll(); + TimeUnit.SECONDS.sleep(4); + + list_a = listenNextEventFor(fd_a, members); + list_b = listenNextEventFor(fd_b, members); + list_c = listenNextEventFor(fd_c, members); + list_d = listenNextEventFor(fd_d, members); + + // Check member D recovers + + assertStatus(a.address(), ALIVE, awaitEvents(list_a), b.address(), c.address(), d.address()); + assertStatus(b.address(), ALIVE, awaitEvents(list_b), a.address(), c.address(), d.address()); + assertStatus(c.address(), ALIVE, awaitEvents(list_c), a.address(), b.address(), d.address()); + assertStatus(d.address(), ALIVE, awaitEvents(list_d), a.address(), b.address(), c.address()); + } finally { + stop(fdetectors); + } + } + + @Test + public void testMemberStatusChangeAfterNetworkRecovery() throws Exception { + // Create transports + Transport a = Transport.bindAwait(true); + Transport b = Transport.bindAwait(true); + List
members = Arrays.asList(a.address(), b.address()); + + // Create failure detectors + FailureDetectorImpl fd_a = createFD(a, members); + FailureDetectorImpl fd_b = createFD(b, members); + List fdetectors = Arrays.asList(fd_a, fd_b); + + // Traffic is blocked initially on both sides: A--X-->B, B--X-->A + a.networkEmulator().block(b.address()); + b.networkEmulator().block(a.address()); + + Future> list_a = listenNextEventFor(fd_a, members); + Future> list_b = listenNextEventFor(fd_b, members); + + try { + start(fdetectors); + + assertStatus(a.address(), SUSPECT, awaitEvents(list_a), b.address()); + assertStatus(b.address(), SUSPECT, awaitEvents(list_b), a.address()); + + // Unblock A and B members: A-->B, B-->A + a.networkEmulator().unblockAll(); + b.networkEmulator().unblockAll(); + TimeUnit.SECONDS.sleep(2); + + // Check that members recover + + list_a = listenNextEventFor(fd_a, members); + list_b = listenNextEventFor(fd_b, members); + + assertStatus(a.address(), ALIVE, awaitEvents(list_a), b.address()); + assertStatus(b.address(), ALIVE, awaitEvents(list_b), a.address()); + } finally { + stop(fdetectors); + } + } + + @Test + public void testStatusChangeAfterMemberRestart() throws Exception { + // Create transports + Transport a = Transport.bindAwait(true); + Transport b = Transport.bindAwait(true); + Transport x = Transport.bindAwait(true); + List
members = Arrays.asList(a.address(), b.address(), x.address()); + + // Create failure detectors + FailureDetectorImpl fd_a = createFD(a, members); + FailureDetectorImpl fd_b = createFD(b, members); + FailureDetectorImpl fd_x = createFD(x, members); + List fdetectors = Arrays.asList(fd_a, fd_b, fd_x); + + Future> list_a = listenNextEventFor(fd_a, members); + Future> list_b = listenNextEventFor(fd_b, members); + Future> list_x = listenNextEventFor(fd_x, members); + + // Restarted member attributes are not initialized + Transport xx; + FailureDetectorImpl fd_xx; + + try { + start(fdetectors); + + assertStatus(a.address(), ALIVE, awaitEvents(list_a), b.address(), x.address()); + assertStatus(b.address(), ALIVE, awaitEvents(list_b), a.address(), x.address()); + assertStatus(x.address(), ALIVE, awaitEvents(list_x), a.address(), b.address()); + + // stop node X + stop(Lists.newArrayList(fd_x)); + TimeUnit.SECONDS.sleep(2); + + // restart node X as XX + xx = Transport.bindAwait(TransportConfig.builder() + .port(x.address().port()) + .portAutoIncrement(false) + .useNetworkEmulator(true) + .build()); + assertEquals(x.address(), xx.address()); + fdetectors = Arrays.asList(fd_a, fd_b, fd_xx = createFD(xx, members)); + + // actual restart here + fd_xx.start(); + TimeUnit.SECONDS.sleep(2); + + list_a = listenNextEventFor(fd_a, members); + list_b = listenNextEventFor(fd_b, members); + Future> list_xx = listenNextEventFor(fd_xx, members); + + // TODO [AK]: It would be more correct to consider restarted member as a new member, so x is still suspected! + + assertStatus(a.address(), ALIVE, awaitEvents(list_a), b.address(), xx.address()); + assertStatus(b.address(), ALIVE, awaitEvents(list_b), a.address(), xx.address()); + assertStatus(xx.address(), ALIVE, awaitEvents(list_xx), a.address(), b.address()); + } finally { + stop(fdetectors); + } + } + + private FailureDetectorImpl createFD(Transport transport, List
members) { + FailureDetectorConfig failureDetectorConfig = ClusterConfig.builder() // faster config for local testing + .pingTimeout(100) + .pingInterval(200) + .pingReqMembers(2) + .build(); + return createFD(transport, members, failureDetectorConfig); + } + + private FailureDetectorImpl createFD(Transport transport, List
addresses, FailureDetectorConfig config) { + MembershipProtocol dummyMembership = new DummyMembershipProtocol(transport.address(), addresses); + return new FailureDetectorImpl(transport, dummyMembership, config); + } + + private void destroyTransport(Transport transport) { + if (transport == null || transport.isStopped()) { + return; + } + CompletableFuture close = new CompletableFuture<>(); + transport.stop(close); + try { + close.get(1, TimeUnit.SECONDS); + } catch (Exception ignore) { + } + } + + private void start(List fdetectors) { + for (FailureDetectorImpl fd : fdetectors) { + fd.start(); + } + } + + private void stop(List fdetectors) { + for (FailureDetectorImpl fd : fdetectors) { + fd.stop(); + } + for (FailureDetectorImpl fd : fdetectors) { + destroyTransport(fd.getTransport()); + } + } + + /** + * @param address target member to expect on + * @param status expected listen status + * @param events events collection of failure detector events + * @param expected expected members of the given listenStatus + */ + private void assertStatus( + Address address, MemberStatus status, Collection events, Address... expected) { + List
actual = events.stream() + .filter(event -> event.status() == status) + .map(FailureDetectorEvent::member) + .map(Member::address) + .collect(Collectors.toList()); + + String msg1 = String.format("Node %s expected %s %s members %s, but was: %s", + address, expected.length, status, Arrays.toString(expected), events); + assertEquals(msg1, expected.length, actual.size()); + + for (Address member : expected) { + String msg2 = String.format("Node %s expected as %s %s, but was: %s", address, status, member, events); + assertTrue(msg2, actual.contains(member)); + } + } + + private Future> listenNextEventFor(FailureDetectorImpl fd, List
addresses) { + addresses = new ArrayList<>(addresses); + addresses.remove(fd.getTransport().address()); // exclude self + checkArgument(!addresses.isEmpty()); + + List> resultFuture = new ArrayList<>(); + for (final Address member : addresses) { + final CompletableFuture future = new CompletableFuture<>(); + fd.listen() + .filter(event -> event.member().address() == member) + .subscribe(future::complete); + resultFuture.add(future); + } + + return allOf(resultFuture); + } + + private Collection awaitEvents(Future> events) { + try { + return events.get(10, TimeUnit.SECONDS); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + private CompletableFuture> allOf(List> futuresList) { + CompletableFuture allFuturesResult = + CompletableFuture.allOf(futuresList.toArray(new CompletableFuture[futuresList.size()])); + return allFuturesResult.thenApply(v -> + futuresList.stream(). + map(CompletableFuture::join). + collect(Collectors.toList()) + ); + } +} diff --git a/cluster/src/test/java/io/scalecube/cluster/gossip/GossipProtocolTest.java b/cluster/src/test/java/io/scalecube/cluster/gossip/GossipProtocolTest.java new file mode 100644 index 00000000..f7ec8cc5 --- /dev/null +++ b/cluster/src/test/java/io/scalecube/cluster/gossip/GossipProtocolTest.java @@ -0,0 +1,267 @@ +package io.scalecube.cluster.gossip; + +import io.scalecube.cluster.ClusterConfig; +import io.scalecube.cluster.ClusterMath; +import io.scalecube.cluster.Member; +import io.scalecube.cluster.membership.DummyMembershipProtocol; +import io.scalecube.cluster.membership.MembershipProtocol; +import io.scalecube.testlib.BaseTest; +import io.scalecube.transport.Transport; +import io.scalecube.transport.Message; +import io.scalecube.transport.Address; +import io.scalecube.transport.TransportConfig; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.LongSummaryStatistics; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; + +import static io.scalecube.cluster.ClusterMath.gossipConvergencePercent; +import static io.scalecube.cluster.ClusterMath.gossipDisseminationTime; +import static io.scalecube.cluster.ClusterMath.maxMessagesPerGossipPerNode; +import static io.scalecube.cluster.ClusterMath.maxMessagesPerGossipTotal; + +@RunWith(Parameterized.class) +public class GossipProtocolTest extends BaseTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(GossipProtocolTest.class); + + private static List experiments = Arrays.asList(new Object[][] { +// N , L , D // N - num of nodes, L - msg loss percent, D - msg mean delay (ms) + { 2 , 0 , 2 }, // warm up + { 2 , 0 , 2 }, + { 3 , 0 , 2 }, + { 5 , 0 , 2 }, + { 10 , 0 , 2 }, + { 10 , 10 , 2 }, + { 10 , 25 , 2 }, + { 10 , 25 , 100 }, + { 10 , 50 , 2 }, + { 50 , 0 , 2 }, + { 50 , 10 , 2 }, + { 50 , 10 , 100 }, + }); + + // Makes tests run longer since always awaits for maximum gossip lifetime, but performs more checks + private static final boolean awaitFullCompletion = true; + + // Allow to configure gossip settings other than defaults + private static final long gossipInterval /* ms */ = ClusterConfig.DEFAULT_GOSSIP_INTERVAL; + private static final int gossipFanout = ClusterConfig.DEFAULT_GOSSIP_FANOUT; + private static final int gossipRepeatMultiplier = ClusterConfig.DEFAULT_GOSSIP_REPEAT_MULT; + + + // Uncomment and modify params to run single experiment repeatedly +// static { +// int repeatCount = 1000; +// int membersNum = 10; +// int lossPercent = 50; //% +// int meanDelay = 2; //ms +// experiments = new ArrayList<>(repeatCount + 1); +// experiments.add(new Object[] {2, 0, 2}); // add warm up experiment +// for (int i = 0; i < repeatCount; i++) { +// experiments.add(new Object[] {membersNum, lossPercent, meanDelay}); +// } +// } + + @Parameterized.Parameters(name = "N={0}, Ploss={1}%, Tmean={2}ms") + public static List data() { + return experiments; + } + + private final int membersNum; + private final int lossPercent; + private final int meanDelay; + + public GossipProtocolTest(int membersNum, int lossPercent, int meanDelay) { + this.membersNum = membersNum; + this.lossPercent = lossPercent; + this.meanDelay = meanDelay; + } + + @Test + public void testGossipProtocol() throws Exception { + // Init gossip protocol instances + List gossipProtocols = initGossipProtocols(membersNum, lossPercent, meanDelay); + + // Subscribe on gossips + long disseminationTime = 0; + LongSummaryStatistics messageSentStatsDissemination = null; + LongSummaryStatistics messageLostStatsDissemination = null; + LongSummaryStatistics messageSentStatsOverall = null; + LongSummaryStatistics messageLostStatsOverall = null; + long gossipTimeout = ClusterMath.gossipTimeoutToSweep(gossipRepeatMultiplier, membersNum, gossipInterval); + try { + final String gossipData = "test gossip - " + ThreadLocalRandom.current().nextLong(); + final CountDownLatch latch = new CountDownLatch(membersNum - 1); + final Map receivers = new ConcurrentHashMap<>(); + final AtomicBoolean doubleDelivery = new AtomicBoolean(false); + for (final GossipProtocolImpl protocol : gossipProtocols) { + protocol.listen().subscribe(gossip -> { + if (gossipData.equals(gossip.data())) { + boolean firstTimeAdded = receivers.put(protocol.getMember(), protocol.getMember()) == null; + if (firstTimeAdded) { + latch.countDown(); + } else { + LOGGER.error("Delivered gossip twice to: {}", protocol.getTransport().address()); + doubleDelivery.set(true); + } + } + }); + } + + // Spread gossip, measure and verify delivery metrics + long start = System.currentTimeMillis(); + gossipProtocols.get(0).spread(Message.fromData(gossipData)); + latch.await(2 * gossipTimeout, TimeUnit.MILLISECONDS); // Await for double gossip timeout + disseminationTime = System.currentTimeMillis() - start; + messageSentStatsDissemination = computeMessageSentStats(gossipProtocols); + if (lossPercent > 0) { + messageLostStatsDissemination = computeMessageLostStats(gossipProtocols); + } + Assert.assertEquals("Not all members received gossip", membersNum - 1, receivers.size()); + Assert.assertTrue("Too long dissemination time " + disseminationTime + + "ms (timeout " + gossipTimeout + "ms)", disseminationTime < gossipTimeout); + + // Await gossip lifetime plus few gossip intervals too ensure gossip is fully spread + if (awaitFullCompletion) { + long awaitCompletionTime = gossipTimeout - disseminationTime + 3 * gossipInterval; + Thread.sleep(awaitCompletionTime); + + messageSentStatsOverall = computeMessageSentStats(gossipProtocols); + if (lossPercent > 0) { + messageLostStatsOverall = computeMessageLostStats(gossipProtocols); + } + } + Assert.assertFalse("Delivered gossip twice to same member", doubleDelivery.get()); + } finally { + // Print theoretical results + LOGGER.info("Experiment params: N={}, Gfanout={}, Grepeat_mult={}, Tgossip={}ms Ploss={}%, Tmean={}ms", + membersNum, gossipFanout, gossipRepeatMultiplier, gossipInterval, lossPercent, meanDelay); + double convergProb = gossipConvergencePercent(gossipFanout, gossipRepeatMultiplier, membersNum, lossPercent); + long expDissemTime = gossipDisseminationTime(gossipRepeatMultiplier, membersNum, gossipInterval); + int maxMsgPerNode = maxMessagesPerGossipPerNode(gossipFanout, gossipRepeatMultiplier, membersNum); + int maxMsgTotal = maxMessagesPerGossipTotal(gossipFanout, gossipRepeatMultiplier, membersNum); + LOGGER.info("Expected dissemination time is {}ms with probability {}%", expDissemTime, convergProb); + LOGGER.info("Max messages sent per node {} and total {}", maxMsgPerNode, maxMsgTotal); + + // Print actual results + LOGGER.info("Actual dissemination time: {}ms (timeout {}ms)", disseminationTime, gossipTimeout); + LOGGER.info("Messages sent stats (diss.): {}", messageSentStatsDissemination); + if (lossPercent > 0) { + LOGGER.info("Messages lost stats (diss.): {}", messageLostStatsDissemination); + } + if (awaitFullCompletion) { + LOGGER.info("Messages sent stats (total): {}", messageSentStatsOverall); + if (lossPercent > 0) { + LOGGER.info("Messages lost stats (total): {}", messageLostStatsOverall); + } + } + + // Destroy gossip protocol instances + destroyGossipProtocols(gossipProtocols); + + } + } + + private LongSummaryStatistics computeMessageSentStats(List gossipProtocols) { + List messageSentPerNode = new ArrayList<>(gossipProtocols.size()); + for (GossipProtocolImpl gossipProtocol : gossipProtocols) { + Transport transport = gossipProtocol.getTransport(); + messageSentPerNode.add(transport.networkEmulator().totalMessageSentCount()); + } + return messageSentPerNode.stream().mapToLong(v -> v).summaryStatistics(); + } + + private LongSummaryStatistics computeMessageLostStats(List gossipProtocols) { + List messageLostPerNode = new ArrayList<>(gossipProtocols.size()); + for (GossipProtocolImpl gossipProtocol : gossipProtocols) { + Transport transport = gossipProtocol.getTransport(); + messageLostPerNode.add(transport.networkEmulator().totalMessageLostCount()); + } + return messageLostPerNode.stream().mapToLong(v -> v).summaryStatistics(); + } + + private List initGossipProtocols(int count, int lostPercent, int meanDelay) { + final List transports = initTransports(count, lostPercent, meanDelay); + List
members = new ArrayList<>(); + for (Transport transport : transports) { + members.add(transport.address()); + } + List gossipProtocols = new ArrayList<>(); + for (Transport transport : transports) { + gossipProtocols.add(initGossipProtocol(transport, members)); + } + return gossipProtocols; + } + + private List initTransports(int count, int lostPercent, int meanDelay) { + List transports = new ArrayList<>(count); + int startPort = TransportConfig.DEFAULT_PORT; + for (int i = 0; i < count; i++) { + TransportConfig transportConfig = TransportConfig.builder() + .useNetworkEmulator(true) + .port(startPort) + .portCount(1000) + .build(); + Transport transport = Transport.bindAwait(transportConfig); + transport.networkEmulator().setDefaultLinkSettings(lostPercent, meanDelay); + transports.add(transport); + startPort = transport.address().port() + 1; + } + return transports; + } + + private GossipProtocolImpl initGossipProtocol(Transport transport, List
members) { + MembershipProtocol dummyMembership = new DummyMembershipProtocol(transport.address(), members); + GossipConfig gossipConfig = ClusterConfig.builder() + .gossipFanout(gossipFanout) + .gossipInterval(gossipInterval) + .gossipRepeatMult(gossipRepeatMultiplier) + .build(); + GossipProtocolImpl gossipProtocol = new GossipProtocolImpl(transport, dummyMembership, gossipConfig); + gossipProtocol.start(); + return gossipProtocol; + } + + private void destroyGossipProtocols(List gossipProtocols) { + // Stop all gossip protocols + for (GossipProtocolImpl gossipProtocol : gossipProtocols) { + gossipProtocol.stop(); + } + + // Stop all transports + List> futures = new ArrayList<>(); + for (GossipProtocolImpl gossipProtocol : gossipProtocols) { + CompletableFuture close = new CompletableFuture<>(); + gossipProtocol.getTransport().stop(close); + futures.add(close); + } + try { + CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()])).get(30, TimeUnit.SECONDS); + } catch (Exception ignore) { + LOGGER.warn("Failed to await transport termination"); + } + + // Await a bit + try { + Thread.sleep(gossipProtocols.size() * 20); + } catch (InterruptedException ignore) { + // ignore + } + } +} diff --git a/cluster/src/test/java/io/scalecube/cluster/gossip/GossipRequestTest.java b/cluster/src/test/java/io/scalecube/cluster/gossip/GossipRequestTest.java new file mode 100644 index 00000000..110895a7 --- /dev/null +++ b/cluster/src/test/java/io/scalecube/cluster/gossip/GossipRequestTest.java @@ -0,0 +1,96 @@ +package io.scalecube.cluster.gossip; + +import static io.netty.buffer.Unpooled.buffer; +import static io.netty.buffer.Unpooled.copiedBuffer; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import io.scalecube.cluster.Member; +import io.scalecube.testlib.BaseTest; +import io.scalecube.transport.Address; +import io.scalecube.transport.Message; + +import io.netty.buffer.ByteBuf; +import io.scalecube.transport.MessageCodec; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class GossipRequestTest extends BaseTest { + + private static final String testDataQualifier = "scalecube/testData"; + + private TestData testData; + + @Before + public void init() throws Throwable { + Map properties = new HashMap<>(); + properties.put("key", "123"); + + testData = new TestData(); + testData.setProperties(properties); + } + + @Test + public void testSerializationAndDeserialization() throws Exception { + + Member from = new Member("0", Address.from("localhost:1234")); + List gossips = getGossips(); + Message message = Message.withData(new GossipRequest(gossips, from.id())).correlationId("CORR_ID").build(); + + ByteBuf bb = buffer(); + MessageCodec.serialize(message, bb); + + assertTrue(bb.readableBytes() > 0); + + ByteBuf input = copiedBuffer(bb); + + Message deserializedMessage = MessageCodec.deserialize(input); + + assertNotNull(deserializedMessage); + Assert.assertEquals(deserializedMessage.data().getClass(), GossipRequest.class); + Assert.assertEquals("CORR_ID", deserializedMessage.correlationId()); + + GossipRequest gossipRequest = deserializedMessage.data(); + assertNotNull(gossipRequest); + assertNotNull(gossipRequest.gossips()); + assertNotNull(gossipRequest.gossips().get(0)); + + Object msgData = gossipRequest.gossips().get(0).message().data(); + assertNotNull(msgData); + assertTrue(msgData.toString(), msgData instanceof TestData); + assertEquals(testData.getProperties(), ((TestData) msgData).getProperties()); + } + + private List getGossips() { + Gossip request = new Gossip("idGossip", Message.withData(testData).qualifier(testDataQualifier).build()); + Gossip request2 = new Gossip("idGossip2", Message.withData(testData).qualifier(testDataQualifier).build()); + List gossips = new ArrayList<>(2); + gossips.add(request); + gossips.add(request2); + return gossips; + } + + private static class TestData { + + private Map properties; + + TestData() {} + + public Map getProperties() { + return properties; + } + + public void setProperties(Map properties) { + this.properties = properties; + } + } + +} diff --git a/cluster/src/test/java/io/scalecube/cluster/membership/DummyMembershipProtocol.java b/cluster/src/test/java/io/scalecube/cluster/membership/DummyMembershipProtocol.java new file mode 100644 index 00000000..cb9f6071 --- /dev/null +++ b/cluster/src/test/java/io/scalecube/cluster/membership/DummyMembershipProtocol.java @@ -0,0 +1,53 @@ +package io.scalecube.cluster.membership; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import io.scalecube.cluster.Member; +import io.scalecube.transport.Address; +import rx.Observable; + +/** + * @author Anton Kharenko + */ +public class DummyMembershipProtocol implements MembershipProtocol { + + private Member localMember; + private List remoteMembers = new ArrayList<>(); + + public DummyMembershipProtocol(Address localAddress, List
allAddresses) { + int count = 0; + for (Address address : allAddresses) { + Member member = new Member(Integer.toString(count++), address); + if (address.equals(localAddress)) { + localMember = member; + } else { + remoteMembers.add(member); + } + } + } + + @Override + public Member member() { + return localMember; + } + + @Override + public void updateMetadata(Map metadata) { + throw new UnsupportedOperationException(); + } + + @Override + public void updateMetadataProperty(String key, String value) { + throw new UnsupportedOperationException(); + } + + @Override + public Observable listen() { + return Observable.from(remoteMembers.stream() + .map(MembershipEvent::createAdded) + .collect(Collectors.toList())); + } +} diff --git a/cluster/src/test/java/io/scalecube/cluster/membership/MembershipProtocolTest.java b/cluster/src/test/java/io/scalecube/cluster/membership/MembershipProtocolTest.java new file mode 100644 index 00000000..bccaf9bb --- /dev/null +++ b/cluster/src/test/java/io/scalecube/cluster/membership/MembershipProtocolTest.java @@ -0,0 +1,598 @@ +package io.scalecube.cluster.membership; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import io.scalecube.cluster.ClusterConfig; +import io.scalecube.cluster.ClusterMath; +import io.scalecube.cluster.fdetector.FailureDetectorImpl; +import io.scalecube.cluster.gossip.GossipProtocolImpl; +import io.scalecube.testlib.BaseTest; +import io.scalecube.transport.Address; +import io.scalecube.transport.Transport; + +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; + +import org.junit.Test; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +public class MembershipProtocolTest extends BaseTest { + + private static final int TEST_PING_INTERVAL = 200; + + @Test + public void testInitialPhaseOk() { + Transport a = Transport.bindAwait(true); + Transport b = Transport.bindAwait(true); + Transport c = Transport.bindAwait(true); + List
members = ImmutableList.of(a.address(), b.address(), c.address()); + + MembershipProtocolImpl cm_a = createMembership(a, members); + MembershipProtocolImpl cm_b = createMembership(b, members); + MembershipProtocolImpl cm_c = createMembership(c, members); + + try { + awaitSeconds(1); + + assertTrusted(cm_a, a.address(), b.address(), c.address()); + assertNoSuspected(cm_a); + assertTrusted(cm_b, a.address(), b.address(), c.address()); + assertNoSuspected(cm_b); + assertTrusted(cm_c, a.address(), b.address(), c.address()); + assertNoSuspected(cm_c); + } finally { + stopAll(cm_a, cm_b, cm_c); + } + } + + @Test + public void testNetworkPartitionThenRecovery() { + Transport a = Transport.bindAwait(true); + Transport b = Transport.bindAwait(true); + Transport c = Transport.bindAwait(true); + List
members = ImmutableList.of(a.address(), b.address(), c.address()); + + MembershipProtocolImpl cm_a = createMembership(a, members); + MembershipProtocolImpl cm_b = createMembership(b, members); + MembershipProtocolImpl cm_c = createMembership(c, members); + + // Block traffic + a.networkEmulator().block(members); + b.networkEmulator().block(members); + c.networkEmulator().block(members); + + try { + awaitSeconds(6); + + assertTrusted(cm_a, a.address()); + assertNoSuspected(cm_a); + assertTrusted(cm_b, b.address()); + assertNoSuspected(cm_b); + assertTrusted(cm_c, c.address()); + assertNoSuspected(cm_c); + + a.networkEmulator().unblockAll(); + b.networkEmulator().unblockAll(); + c.networkEmulator().unblockAll(); + + awaitSeconds(6); + + assertTrusted(cm_a, a.address(), b.address(), c.address()); + assertNoSuspected(cm_a); + assertTrusted(cm_b, a.address(), b.address(), c.address()); + assertNoSuspected(cm_b); + assertTrusted(cm_c, a.address(), b.address(), c.address()); + assertNoSuspected(cm_c); + } finally { + stopAll(cm_a, cm_b, cm_c); + } + } + + @Test + public void testMemberLostNetworkThenRecover() { + Transport a = Transport.bindAwait(true); + Transport b = Transport.bindAwait(true); + Transport c = Transport.bindAwait(true); + List
members = ImmutableList.of(a.address(), b.address(), c.address()); + + MembershipProtocolImpl cm_a = createMembership(a, members); + MembershipProtocolImpl cm_b = createMembership(b, members); + MembershipProtocolImpl cm_c = createMembership(c, members); + + try { + awaitSeconds(1); + + // Check all trusted + assertTrusted(cm_a, a.address(), b.address(), c.address()); + assertNoSuspected(cm_a); + assertTrusted(cm_b, a.address(), b.address(), c.address()); + assertNoSuspected(cm_b); + assertTrusted(cm_c, a.address(), b.address(), c.address()); + assertNoSuspected(cm_c); + + // Node b lost network + b.networkEmulator().block(Arrays.asList(a.address(), c.address())); + a.networkEmulator().block(b.address()); + c.networkEmulator().block(b.address()); + + awaitSeconds(1); + + // Check partition: {b}, {a, c} + assertTrusted(cm_a, a.address(), c.address()); + assertSuspected(cm_a, b.address()); + assertTrusted(cm_b, b.address()); + assertSuspected(cm_b, a.address(), c.address()); + assertTrusted(cm_c, a.address(), c.address()); + assertSuspected(cm_c, b.address()); + + // Node b recover network + a.networkEmulator().unblockAll(); + b.networkEmulator().unblockAll(); + c.networkEmulator().unblockAll(); + + awaitSeconds(1); + + // Check all trusted again + assertTrusted(cm_a, a.address(), b.address(), c.address()); + assertNoSuspected(cm_a); + assertTrusted(cm_b, a.address(), b.address(), c.address()); + assertNoSuspected(cm_b); + assertTrusted(cm_c, a.address(), b.address(), c.address()); + assertNoSuspected(cm_c); + } finally { + stopAll(cm_a, cm_b, cm_c); + } + } + + @Test + public void testDoublePartitionThenRecover() { + Transport a = Transport.bindAwait(true); + Transport b = Transport.bindAwait(true); + Transport c = Transport.bindAwait(true); + List
members = ImmutableList.of(a.address(), b.address(), c.address()); + + MembershipProtocolImpl cm_a = createMembership(a, members); + MembershipProtocolImpl cm_b = createMembership(b, members); + MembershipProtocolImpl cm_c = createMembership(c, members); + + try { + awaitSeconds(1); + + // Check all trusted + assertTrusted(cm_a, a.address(), b.address(), c.address()); + assertNoSuspected(cm_a); + assertTrusted(cm_b, a.address(), b.address(), c.address()); + assertNoSuspected(cm_b); + assertTrusted(cm_c, a.address(), b.address(), c.address()); + assertNoSuspected(cm_c); + + // Node b lost network + b.networkEmulator().block(Arrays.asList(a.address(), c.address())); + a.networkEmulator().block(b.address()); + c.networkEmulator().block(b.address()); + + awaitSeconds(1); + + // Check partition: {b}, {a, c} + assertTrusted(cm_a, a.address(), c.address()); + assertSuspected(cm_a, b.address()); + assertTrusted(cm_b, b.address()); + assertSuspected(cm_b, a.address(), c.address()); + assertTrusted(cm_c, a.address(), c.address()); + assertSuspected(cm_c, b.address()); + + // Node a and c lost network + a.networkEmulator().block(c.address()); + c.networkEmulator().block(a.address()); + + awaitSeconds(1); + + // Check partition: {a}, {b}, {c} + assertTrusted(cm_a, a.address()); + assertSuspected(cm_a, b.address(), c.address()); + assertTrusted(cm_b, b.address()); + assertSuspected(cm_b, a.address(), c.address()); + assertTrusted(cm_c, c.address()); + assertSuspected(cm_c, b.address(), a.address()); + + // Recover network + a.networkEmulator().unblockAll(); + b.networkEmulator().unblockAll(); + c.networkEmulator().unblockAll(); + + awaitSeconds(1); + + // Check all trusted again + assertTrusted(cm_a, a.address(), b.address(), c.address()); + assertNoSuspected(cm_a); + assertTrusted(cm_b, a.address(), b.address(), c.address()); + assertNoSuspected(cm_b); + assertTrusted(cm_c, a.address(), b.address(), c.address()); + assertNoSuspected(cm_c); + } finally { + stopAll(cm_a, cm_b, cm_c); + } + } + + @Test + public void testNetworkDisabledThenRecovered() { + Transport a = Transport.bindAwait(true); + Transport b = Transport.bindAwait(true); + Transport c = Transport.bindAwait(true); + List
members = ImmutableList.of(a.address(), b.address(), c.address()); + + MembershipProtocolImpl cm_a = createMembership(a, members); + MembershipProtocolImpl cm_b = createMembership(b, members); + MembershipProtocolImpl cm_c = createMembership(c, members); + + try { + awaitSeconds(1); + + assertTrusted(cm_a, a.address(), b.address(), c.address()); + assertNoSuspected(cm_a); + assertTrusted(cm_b, a.address(), b.address(), c.address()); + assertNoSuspected(cm_b); + assertTrusted(cm_c, a.address(), b.address(), c.address()); + assertNoSuspected(cm_c); + + a.networkEmulator().block(members); + b.networkEmulator().block(members); + c.networkEmulator().block(members); + + awaitSeconds(1); + + assertTrusted(cm_a, a.address()); + assertSuspected(cm_a, b.address(), c.address()); + + assertTrusted(cm_b, b.address()); + assertSuspected(cm_b,a.address(), c.address()); + + assertTrusted(cm_c, c.address()); + assertSuspected(cm_c, a.address(), b.address()); + + a.networkEmulator().unblockAll(); + b.networkEmulator().unblockAll(); + c.networkEmulator().unblockAll(); + + awaitSeconds(1); + + assertTrusted(cm_a, a.address(), b.address(), c.address()); + assertNoSuspected(cm_a); + + assertTrusted(cm_b, a.address(), b.address(), c.address()); + assertNoSuspected(cm_b); + + assertTrusted(cm_c, a.address(), b.address(), c.address()); + assertNoSuspected(cm_c); + } finally { + stopAll(cm_a, cm_b, cm_c); + } + } + + @Test + public void testLongNetworkPartitionNoRecovery() { + Transport a = Transport.bindAwait(true); + Transport b = Transport.bindAwait(true); + Transport c = Transport.bindAwait(true); + Transport d = Transport.bindAwait(true); + List
members = ImmutableList.of(a.address(), b.address(), c.address(), d.address()); + + MembershipProtocolImpl cm_a = createMembership(a, members); + MembershipProtocolImpl cm_b = createMembership(b, members); + MembershipProtocolImpl cm_c = createMembership(c, members); + MembershipProtocolImpl cm_d = createMembership(d, members); + + try { + awaitSeconds(1); + + assertTrusted(cm_a, a.address(), b.address(), c.address(), d.address()); + assertTrusted(cm_b, a.address(), b.address(), c.address(), d.address()); + assertTrusted(cm_c, a.address(), b.address(), c.address(), d.address()); + assertTrusted(cm_d, a.address(), b.address(), c.address(), d.address()); + + a.networkEmulator().block(Arrays.asList(c.address(), d.address())); + b.networkEmulator().block(Arrays.asList(c.address(), d.address())); + + c.networkEmulator().block(Arrays.asList(a.address(), b.address())); + d.networkEmulator().block(Arrays.asList(a.address(), b.address())); + + awaitSeconds(2); + + assertTrusted(cm_a, a.address(), b.address()); + assertSuspected(cm_a, c.address(), d.address()); + assertTrusted(cm_b, a.address(), b.address()); + assertSuspected(cm_b, c.address(), d.address()); + assertTrusted(cm_c, c.address(), d.address()); + assertSuspected(cm_c, a.address(), b.address()); + assertTrusted(cm_d, c.address(), d.address()); + assertSuspected(cm_d, a.address(), b.address()); + + long suspicionTimeoutSec = + ClusterMath.suspicionTimeout(ClusterConfig.DEFAULT_SUSPICION_MULT, 4, TEST_PING_INTERVAL) / 1000; + awaitSeconds(suspicionTimeoutSec + 1); // > max suspect time + + assertTrusted(cm_a, a.address(), b.address()); + assertNoSuspected(cm_a); + assertTrusted(cm_b, a.address(), b.address()); + assertNoSuspected(cm_b); + assertTrusted(cm_c, c.address(), d.address()); + assertNoSuspected(cm_c); + assertTrusted(cm_d, c.address(), d.address()); + assertNoSuspected(cm_d); + } finally { + stopAll(cm_a, cm_b, cm_c, cm_d); + } + } + + @Test + public void testRestartFailedMembers() { + Transport a = Transport.bindAwait(true); + Transport b = Transport.bindAwait(true); + Transport c = Transport.bindAwait(true); + Transport d = Transport.bindAwait(true); + List
members = ImmutableList.of(a.address(), b.address(), c.address(), d.address()); + + MembershipProtocolImpl cm_a = createMembership(a, members); + MembershipProtocolImpl cm_b = createMembership(b, members); + MembershipProtocolImpl cm_c = createMembership(c, members); + MembershipProtocolImpl cm_d = createMembership(d, members); + + MembershipProtocolImpl cm_restartedC = null; + MembershipProtocolImpl cm_restartedD = null; + + try { + awaitSeconds(1); + + assertTrusted(cm_a, a.address(), b.address(), c.address(), d.address()); + assertTrusted(cm_b, a.address(), b.address(), c.address(), d.address()); + assertTrusted(cm_c, a.address(), b.address(), c.address(), d.address()); + assertTrusted(cm_d, a.address(), b.address(), c.address(), d.address()); + + stop(cm_c); + stop(cm_d); + + awaitSeconds(1); + + assertTrusted(cm_a, a.address(), b.address()); + assertSuspected(cm_a, c.address(), d.address()); + assertTrusted(cm_b, a.address(), b.address()); + assertSuspected(cm_b, c.address(), d.address()); + + long suspicionTimeoutSec = + ClusterMath.suspicionTimeout(ClusterConfig.DEFAULT_SUSPICION_MULT, 4, TEST_PING_INTERVAL) / 1000; + awaitSeconds(suspicionTimeoutSec + 1); // > max suspect time + + assertTrusted(cm_a, a.address(), b.address()); + assertNoSuspected(cm_a); + assertTrusted(cm_b, a.address(), b.address()); + assertNoSuspected(cm_b); + + c = Transport.bindAwait(true); + d = Transport.bindAwait(true); + cm_restartedC = createMembership(c, Arrays.asList(a.address(), b.address())); + cm_restartedD = createMembership(d, Arrays.asList(a.address(), b.address())); + + awaitSeconds(1); + + assertTrusted(cm_restartedC, a.address(), b.address(), c.address(), d.address()); + assertNoSuspected(cm_restartedC); + assertTrusted(cm_restartedD, a.address(), b.address(), c.address(), d.address()); + assertNoSuspected(cm_restartedD); + assertTrusted(cm_a, a.address(), b.address(), c.address(), d.address()); + assertNoSuspected(cm_a); + assertTrusted(cm_b, a.address(), b.address(), c.address(), d.address()); + assertNoSuspected(cm_b); + } finally { + stopAll(cm_a, cm_b, cm_restartedC, cm_restartedD); + } + } + + @Test + public void testLimitedSeedMembers() { + Transport a = Transport.bindAwait(true); + Transport b = Transport.bindAwait(true); + Transport c = Transport.bindAwait(true); + Transport d = Transport.bindAwait(true); + Transport e = Transport.bindAwait(true); + + MembershipProtocolImpl cm_a = createMembership(a, Collections.emptyList()); + MembershipProtocolImpl cm_b = createMembership(b, Collections.singletonList(a.address())); + MembershipProtocolImpl cm_c = createMembership(c, Collections.singletonList(a.address())); + MembershipProtocolImpl cm_d = createMembership(d, Collections.singletonList(b.address())); + MembershipProtocolImpl cm_e = createMembership(e, Collections.singletonList(b.address())); + + try { + awaitSeconds(3); + + assertTrusted(cm_a, a.address(), b.address(), c.address(), d.address(), e.address()); + assertNoSuspected(cm_a); + assertTrusted(cm_b, a.address(), b.address(), c.address(), d.address(), e.address()); + assertNoSuspected(cm_b); + assertTrusted(cm_c, a.address(), b.address(), c.address(), d.address(), e.address()); + assertNoSuspected(cm_c); + assertTrusted(cm_d, a.address(), b.address(), c.address(), d.address(), e.address()); + assertNoSuspected(cm_d); + assertTrusted(cm_e, a.address(), b.address(), c.address(), d.address(), e.address()); + assertNoSuspected(cm_e); + } finally { + stopAll(cm_a, cm_b, cm_c, cm_d, cm_e); + } + } + + @Test + public void testOverrideMemberAddress() throws UnknownHostException { + String localAddress = InetAddress.getLocalHost().getHostName(); + + Transport a = Transport.bindAwait(true); + Transport b = Transport.bindAwait(true); + Transport c = Transport.bindAwait(true); + Transport d = Transport.bindAwait(true); + Transport e = Transport.bindAwait(true); + + MembershipProtocolImpl cm_a = createMembership(a, testConfig(Collections.emptyList()).memberHost(localAddress).build()); + MembershipProtocolImpl cm_b = createMembership(b, testConfig(Collections.singletonList(a.address())).memberHost(localAddress).build()); + MembershipProtocolImpl cm_c = createMembership(c, testConfig(Collections.singletonList(a.address())).memberHost(localAddress).build()); + MembershipProtocolImpl cm_d = createMembership(d, testConfig(Collections.singletonList(b.address())).memberHost(localAddress).build()); + MembershipProtocolImpl cm_e = createMembership(e, testConfig(Collections.singletonList(b.address())).memberHost(localAddress).build()); + + try { + awaitSeconds(3); + + assertTrusted(cm_a, cm_a.member().address(), cm_b.member().address(), cm_c.member().address(), cm_d.member().address(), cm_e.member().address()); + assertNoSuspected(cm_a); + assertTrusted(cm_b, cm_a.member().address(), cm_b.member().address(), cm_c.member().address(), cm_d.member().address(), cm_e.member().address()); + assertNoSuspected(cm_b); + assertTrusted(cm_c, cm_a.member().address(), cm_b.member().address(), cm_c.member().address(), cm_d.member().address(), cm_e.member().address()); + assertNoSuspected(cm_c); + assertTrusted(cm_d, cm_a.member().address(), cm_b.member().address(), cm_c.member().address(), cm_d.member().address(), cm_e.member().address()); + assertNoSuspected(cm_d); + assertTrusted(cm_e, cm_a.member().address(), cm_b.member().address(), cm_c.member().address(), cm_d.member().address(), cm_e.member().address()); + assertNoSuspected(cm_e); + } finally { + stopAll(cm_a, cm_b, cm_c, cm_d, cm_e); + } + } + + @Test + public void testMemberAddressOverrides() { + Transport t = Transport.bindAwait(true); + String host = "host1"; + + // Default behavior + Address address = MembershipProtocolImpl.memberAddress(t, + testConfig(Collections.emptyList()) + .build()); + assertEquals(t.address(), address); + + // Override host only + address = MembershipProtocolImpl.memberAddress(t, + testConfig(Collections.emptyList()) + .memberHost(host) + .build()); + assertEquals(Address.create(host, t.address().port()), address); + + // Override host and port + address = MembershipProtocolImpl.memberAddress(t, + testConfig(Collections.emptyList()) + .memberHost(host).memberPort(80) + .build()); + assertEquals(Address.create(host, 80), address); + + // Override port only (override is ignored) + address = MembershipProtocolImpl.memberAddress(t, + testConfig(Collections.emptyList()) + .memberPort(8080) + .build()); + assertEquals(t.address(), address); + } + + private void awaitSeconds(long seconds) { + try { + TimeUnit.SECONDS.sleep(seconds); + } catch (InterruptedException e) { + Throwables.propagate(e); + } + } + + private ClusterConfig overrideConfig(Address seedAddress, String memberHost) { + return testConfig(seedAddress != null + ? Collections.singletonList(seedAddress) + : Collections.emptyList() + ).memberHost(memberHost).build(); + } + + private ClusterConfig.Builder testConfig(List
seedAddresses) { + // Create faster config for local testing + return ClusterConfig.builder() + .seedMembers(seedAddresses) + .syncInterval(2000) + .syncTimeout(1000) + .pingInterval(TEST_PING_INTERVAL) + .pingTimeout(100); + } + + private MembershipProtocolImpl createMembership(Transport transport, List
seedAddresses) { + return createMembership(transport, testConfig(seedAddresses).build()); + } + + private MembershipProtocolImpl createMembership(Transport transport, ClusterConfig config) { + // Create components + MembershipProtocolImpl membership = new MembershipProtocolImpl(transport, config); + FailureDetectorImpl failureDetector = new FailureDetectorImpl(transport, membership, config); + GossipProtocolImpl gossipProtocol = new GossipProtocolImpl(transport, membership, config); + membership.setGossipProtocol(gossipProtocol); + membership.setFailureDetector(failureDetector); + + try { + failureDetector.start(); + gossipProtocol.start(); + membership.start().get(); + } catch (Exception ex) { + Throwables.propagate(ex); + } + + return membership; + } + + private void stopAll(MembershipProtocolImpl... memberships) { + for (MembershipProtocolImpl membership : memberships) { + if (membership != null) { + stop(membership); + } + } + } + + private void stop(MembershipProtocolImpl membership) { + membership.stop(); + membership.getGossipProtocol().stop(); + membership.getFailureDetector().stop(); + + Transport transport = membership.getTransport(); + CompletableFuture close = new CompletableFuture<>(); + transport.stop(close); + try { + close.get(1, TimeUnit.SECONDS); + } catch (Exception ignore) { + // ignore + } + } + + private void assertTrusted(MembershipProtocolImpl membership, Address... expected) { + List
actual = getAddressesWithStatus(membership, MemberStatus.ALIVE); + assertEquals("Expected " + expected.length + " trusted members " + Arrays.toString(expected) + + ", but actual: " + actual, expected.length, actual.size()); + for (Address member : expected) { + assertTrue("Expected to trust " + member + ", but actual: " + actual, actual.contains(member)); + } + } + + private void assertSuspected(MembershipProtocolImpl membership, Address... expected) { + List
actual = getAddressesWithStatus(membership, MemberStatus.SUSPECT); + assertEquals("Expected " + expected.length + " suspect members " + Arrays.toString(expected) + + ", but actual: " + actual, expected.length, actual.size()); + for (Address member : expected) { + assertTrue("Expected to suspect " + member + ", but actual: " + actual, actual.contains(member)); + } + } + + private void assertNoSuspected(MembershipProtocolImpl membership) { + List
actual = getAddressesWithStatus(membership, MemberStatus.SUSPECT); + assertEquals("Expected no suspected, but actual: " + actual, 0, actual.size()); + } + + private List
getAddressesWithStatus(MembershipProtocolImpl membership, MemberStatus status) { + return membership.getMembershipRecords().stream() + .filter(member -> member.status() == status) + .map(MembershipRecord::address) + .collect(Collectors.toList()); + } +} diff --git a/cluster/src/test/java/io/scalecube/cluster/membership/MembershipRecordTest.java b/cluster/src/test/java/io/scalecube/cluster/membership/MembershipRecordTest.java new file mode 100644 index 00000000..564889be --- /dev/null +++ b/cluster/src/test/java/io/scalecube/cluster/membership/MembershipRecordTest.java @@ -0,0 +1,106 @@ +package io.scalecube.cluster.membership; + +import org.junit.Test; + +import io.scalecube.cluster.Member; +import io.scalecube.testlib.BaseTest; +import io.scalecube.transport.Address; + +import static io.scalecube.cluster.membership.MemberStatus.ALIVE; +import static io.scalecube.cluster.membership.MemberStatus.DEAD; +import static io.scalecube.cluster.membership.MemberStatus.SUSPECT; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +public class MembershipRecordTest extends BaseTest { + + private final Member member = new Member("0", Address.from("localhost:1234")); + private final Member anotherMember = new Member("1", Address.from("localhost:4567")); + + private final MembershipRecord r0_null = null; + + private final MembershipRecord r0_alive_0 = new MembershipRecord(member, ALIVE, 0); + private final MembershipRecord r0_alive_1 = new MembershipRecord(member, ALIVE, 1); + private final MembershipRecord r0_alive_2 = new MembershipRecord(member, ALIVE, 2); + + private final MembershipRecord r0_suspect_0 = new MembershipRecord(member, SUSPECT, 0); + private final MembershipRecord r0_suspect_1 = new MembershipRecord(member, SUSPECT, 1); + private final MembershipRecord r0_suspect_2 = new MembershipRecord(member, SUSPECT, 2); + + private final MembershipRecord r0_dead_0 = new MembershipRecord(member, DEAD, 0); + private final MembershipRecord r0_dead_1 = new MembershipRecord(member, DEAD, 1); + private final MembershipRecord r0_dead_2 = new MembershipRecord(member, DEAD, 2); + + @Test(expected = IllegalArgumentException.class) + public void testCantCompareDifferentMembers() { + MembershipRecord r0 = new MembershipRecord(member, ALIVE, 0); + MembershipRecord r1 = new MembershipRecord(anotherMember, ALIVE, 0); + + r1.isOverrides(r0); // throws exception + } + + @Test + public void testDeadOverride() { + MembershipRecord r1_dead_1 = new MembershipRecord(member, DEAD, 1); + + assertFalse(r1_dead_1.isOverrides(r0_null)); + + assertTrue(r1_dead_1.isOverrides(r0_alive_0)); + assertTrue(r1_dead_1.isOverrides(r0_alive_1)); + assertTrue(r1_dead_1.isOverrides(r0_alive_2)); + + assertTrue(r1_dead_1.isOverrides(r0_suspect_0)); + assertTrue(r1_dead_1.isOverrides(r0_suspect_1)); + assertTrue(r1_dead_1.isOverrides(r0_suspect_2)); + + assertFalse(r1_dead_1.isOverrides(r0_dead_0)); + assertFalse(r1_dead_1.isOverrides(r0_dead_1)); + assertFalse(r1_dead_1.isOverrides(r0_dead_2)); + } + + @Test + public void testAliveOverride() { + MembershipRecord r1_alive_1 = new MembershipRecord(member, ALIVE, 1); + + assertTrue(r1_alive_1.isOverrides(r0_null)); + + assertTrue(r1_alive_1.isOverrides(r0_alive_0)); + assertFalse(r1_alive_1.isOverrides(r0_alive_1)); + assertFalse(r1_alive_1.isOverrides(r0_alive_2)); + + assertTrue(r1_alive_1.isOverrides(r0_suspect_0)); + assertFalse(r1_alive_1.isOverrides(r0_suspect_1)); + assertFalse(r1_alive_1.isOverrides(r0_suspect_2)); + + assertFalse(r1_alive_1.isOverrides(r0_dead_0)); + assertFalse(r1_alive_1.isOverrides(r0_dead_1)); + assertFalse(r1_alive_1.isOverrides(r0_dead_2)); + } + + @Test + public void testSuspectOverride() { + MembershipRecord r1_suspect_1 = new MembershipRecord(member, SUSPECT, 1); + + assertFalse(r1_suspect_1.isOverrides(r0_null)); + + assertTrue(r1_suspect_1.isOverrides(r0_alive_0)); + assertTrue(r1_suspect_1.isOverrides(r0_alive_1)); + assertFalse(r1_suspect_1.isOverrides(r0_alive_2)); + + assertTrue(r1_suspect_1.isOverrides(r0_suspect_0)); + assertFalse(r1_suspect_1.isOverrides(r0_suspect_1)); + assertFalse(r1_suspect_1.isOverrides(r0_suspect_2)); + + assertFalse(r1_suspect_1.isOverrides(r0_dead_0)); + assertFalse(r1_suspect_1.isOverrides(r0_dead_1)); + assertFalse(r1_suspect_1.isOverrides(r0_dead_2)); + } + + @Test + public void testEqualRecordNotOverriding() { + assertFalse(r0_alive_1.isOverrides(r0_alive_1)); + assertFalse(r0_suspect_1.isOverrides(r0_suspect_1)); + assertFalse(r0_dead_1.isOverrides(r0_dead_1)); + } + +} diff --git a/eclipse-java-google-style.xml b/eclipse-java-google-style.xml new file mode 100644 index 00000000..83c9af0f --- /dev/null +++ b/eclipse-java-google-style.xml @@ -0,0 +1,730 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/eclipse.importorder b/eclipse.importorder new file mode 100644 index 00000000..f713aff3 --- /dev/null +++ b/eclipse.importorder @@ -0,0 +1,9 @@ +#Organize Import Order +#Fri Apr 13 21:16:47 IDT 2018 +6=javax +5=java +4=rx +3=org +2=io +1=com +0=io.scalecube diff --git a/examples/pom.xml b/examples/pom.xml new file mode 100644 index 00000000..1d2bc67e --- /dev/null +++ b/examples/pom.xml @@ -0,0 +1,24 @@ + + 4.0.0 + + io.scalecube + scalecube-parent + 1.0.9-SNAPSHOT + + scalecube-examples + ScaleCube/Examples + + + + ${project.groupId} + scalecube-services + ${project.version} + + + io.scalecube + scalecube-gateway + ${project.version} + + + + diff --git a/examples/src/main/java/io/scalecube/examples/ClusterJoinExamples.java b/examples/src/main/java/io/scalecube/examples/ClusterJoinExamples.java new file mode 100644 index 00000000..7b9ac15a --- /dev/null +++ b/examples/src/main/java/io/scalecube/examples/ClusterJoinExamples.java @@ -0,0 +1,67 @@ +package io.scalecube.examples; + +import static java.util.stream.Collectors.joining; + +import io.scalecube.cluster.Cluster; +import io.scalecube.cluster.ClusterConfig; +import io.scalecube.cluster.Member; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +/** + * Example how to create {@link Cluster} instances and join them to cluster. + * + * @author Anton Kharenko + */ +public class ClusterJoinExamples { + + /** + * Main method. + */ + public static void main(String[] args) throws Exception { + // Start seed member Alice + Cluster alice = Cluster.joinAwait(); + + // Join Bob to cluster with Alice + Cluster bob = Cluster.joinAwait(alice.address()); + + // Join Carol to cluster with metadata + Map metadata = ImmutableMap.of("name", "Carol"); + Cluster carol = Cluster.joinAwait(metadata, alice.address()); + + // Start Dan on port 3000 + ClusterConfig configWithFixedPort = ClusterConfig.builder() + .seedMembers(alice.address()) + .portAutoIncrement(false) + .port(3000) + .build(); + Cluster dan = Cluster.joinAwait(configWithFixedPort); + + // Start Eve in separate cluster (separate sync group) + ClusterConfig configWithSyncGroup = ClusterConfig.builder() + .seedMembers(alice.address(), bob.address(), carol.address(), dan.address()) // won't join anyway + .syncGroup("another cluster") + .build(); + Cluster eve = Cluster.joinAwait(configWithSyncGroup); + + // Print cluster members of each node + + System.out.println("Alice (" + alice.address() + ") cluster: " + + alice.members().stream().map(Member::toString).collect(joining("\n", "\n", "\n"))); + + System.out.println("Bob (" + bob.address() + ") cluster: " + + bob.members().stream().map(Member::toString).collect(joining("\n", "\n", "\n"))); + + System.out.println("Carol (" + carol.address() + ") cluster: " + + carol.members().stream().map(Member::toString).collect(joining("\n", "\n", "\n"))); + + System.out.println("Dan (" + dan.address() + ") cluster: " + + dan.members().stream().map(Member::toString).collect(joining("\n", "\n", "\n"))); + + System.out.println("Eve (" + eve.address() + ") cluster: " // alone in cluster + + eve.members().stream().map(Member::toString).collect(joining("\n", "\n", "\n"))); + } + +} diff --git a/examples/src/main/java/io/scalecube/examples/ClusterMetadataExample.java b/examples/src/main/java/io/scalecube/examples/ClusterMetadataExample.java new file mode 100644 index 00000000..7b681fc9 --- /dev/null +++ b/examples/src/main/java/io/scalecube/examples/ClusterMetadataExample.java @@ -0,0 +1,54 @@ +package io.scalecube.examples; + +import io.scalecube.cluster.Cluster; +import io.scalecube.cluster.Member; +import io.scalecube.transport.Message; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +/** + * Using Cluster metadata: metadata is set of custom parameters that may be used by application developers to attach + * additional business information and identifications to cluster members. + * + *

+ * in this example we see how to attach logical name to a cluster member we nick name Joe + *

+ * + * @author ronen_h, Anton Kharenko + */ +public class ClusterMetadataExample { + + /** + * Main method. + */ + public static void main(String[] args) throws Exception { + // Start seed cluster member Alice + Cluster alice = Cluster.joinAwait(); + + // Join Joe to cluster with metadata + Map metadata = ImmutableMap.of("name", "Joe"); + Cluster joe = Cluster.joinAwait(metadata, alice.address()); + + // Subscribe Joe to listen for incoming messages and print them to system out + joe.listen() + .map(Message::data) + .subscribe(System.out::println); + + // Scan the list of members in the cluster and find Joe there + Optional joeMemberOptional = alice.otherMembers().stream() + .filter(member -> "Joe".equals(member.metadata().get("name"))) + .findAny(); + + // Send hello to Joe + if (joeMemberOptional.isPresent()) { + alice.send(joeMemberOptional.get(), Message.fromData("Hello Joe")); + } + + TimeUnit.SECONDS.sleep(3); + } + +} diff --git a/examples/src/main/java/io/scalecube/examples/GossipExample.java b/examples/src/main/java/io/scalecube/examples/GossipExample.java new file mode 100644 index 00000000..077a1ed4 --- /dev/null +++ b/examples/src/main/java/io/scalecube/examples/GossipExample.java @@ -0,0 +1,39 @@ +package io.scalecube.examples; + +import io.scalecube.cluster.Cluster; +import io.scalecube.transport.Message; + +/** + * Basic example for member gossiping between cluster members. to run the example Start ClusterNodeA and cluster + * ClusterNodeB A listen on gossip B spread gossip + * + * @author ronen hamias, Anton Kharenko + * + */ +public class GossipExample { + + /** + * Main method. + */ + public static void main(String[] args) throws Exception { + // Start cluster nodes and subscribe on listening gossips + Cluster alice = Cluster.joinAwait(); + alice.listenGossips().subscribe(gossip -> System.out.println("Alice heard: " + gossip.data())); + + Cluster bob = Cluster.joinAwait(alice.address()); + bob.listenGossips().subscribe(gossip -> System.out.println("Bob heard: " + gossip.data())); + + Cluster carol = Cluster.joinAwait(alice.address()); + carol.listenGossips().subscribe(gossip -> System.out.println("Carol heard: " + gossip.data())); + + Cluster dan = Cluster.joinAwait(alice.address()); + dan.listenGossips().subscribe(gossip -> System.out.println("Dan heard: " + gossip.data())); + + // Start cluster node Eve that joins cluster and spreads gossip + Cluster eve = Cluster.joinAwait(alice.address()); + eve.spreadGossip(Message.fromData("Gossip from Eve")); + + // Avoid exit main thread immediately ]:-> + Thread.sleep(1000); + } +} diff --git a/examples/src/main/java/io/scalecube/examples/MembershipEventsExample.java b/examples/src/main/java/io/scalecube/examples/MembershipEventsExample.java new file mode 100644 index 00000000..3f18843e --- /dev/null +++ b/examples/src/main/java/io/scalecube/examples/MembershipEventsExample.java @@ -0,0 +1,60 @@ +package io.scalecube.examples; + +import io.scalecube.cluster.Cluster; +import io.scalecube.cluster.ClusterConfig; +import io.scalecube.cluster.ClusterMath; + +import com.google.common.collect.ImmutableMap; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.concurrent.Future; + +/** + * Example of subscribing and listening for cluster membership events which is emmited when new member joins or leave + * cluster. + * + * @author Anton Kharenko + */ +public class MembershipEventsExample { + + private static final SimpleDateFormat sdf = new SimpleDateFormat("HH:mm:ss.SSS"); + + /** + * Main method. + */ + public static void main(String[] args) throws Exception { + // Alice init cluster + Cluster alice = Cluster.joinAwait(ImmutableMap.of("name", "Alice")); + System.out.println(now() + " Alice join members: " + alice.members()); + alice.listenMembership() + .subscribe(event -> System.out.println(now() + " Alice received: " + event)); + + // Bob join cluster + Cluster bob = Cluster.joinAwait(ImmutableMap.of("name", "Bob"), alice.address()); + System.out.println(now() + " Bob join members: " + bob.members()); + bob.listenMembership() + .subscribe(event -> System.out.println(now() + " Bob received: " + event)); + + // Carol join cluster + Cluster carol = Cluster.joinAwait(ImmutableMap.of("name", "Carol"), alice.address(), bob.address()); + System.out.println(now() + " Carol join members: " + carol.members()); + carol.listenMembership() + .subscribe(event -> System.out.println(now() + " Carol received: " + event)); + + // Bob leave cluster + Future shutdownFuture = bob.shutdown(); + shutdownFuture.get(); + + // Avoid exit main thread immediately ]:-> + long pingInterval = ClusterConfig.DEFAULT_PING_INTERVAL; + long suspicionTimeout = ClusterMath.suspicionTimeout(ClusterConfig.DEFAULT_SUSPICION_MULT, 4, pingInterval); + long maxRemoveTimeout = suspicionTimeout + 3 * pingInterval; + Thread.sleep(maxRemoveTimeout); + } + + private static String now() { + return sdf.format(new Date()); + } + +} diff --git a/examples/src/main/java/io/scalecube/examples/MessagingExample.java b/examples/src/main/java/io/scalecube/examples/MessagingExample.java new file mode 100644 index 00000000..8d4ef1aa --- /dev/null +++ b/examples/src/main/java/io/scalecube/examples/MessagingExample.java @@ -0,0 +1,49 @@ +package io.scalecube.examples; + +import io.scalecube.cluster.Cluster; +import io.scalecube.transport.Message; + +/** + * Basic example for member transport between cluster members to run the example Start ClusterNodeA and cluster + * ClusterNodeB A listen on transport messages B send message to member A. + * + * @author ronen hamias, Anton Kharenko + * + */ +public class MessagingExample { + + /** + * Main method. + */ + public static void main(String[] args) throws Exception { + // Start cluster node Alice to listen and respond for incoming greeting messages + Cluster alice = Cluster.joinAwait(); + alice.listen().subscribe(msg -> { + System.out.println("Alice received: " + msg.data()); + alice.send(msg.sender(), Message.fromData("Greetings from Alice")); + }); + + // Join cluster node Bob to cluster with Alice, listen and respond for incoming greeting messages + Cluster bob = Cluster.joinAwait(alice.address()); + bob.listen().subscribe(msg -> { + System.out.println("Bob received: " + msg.data()); + bob.send(msg.sender(), Message.fromData("Greetings from Bob")); + }); + + // Join cluster node Carol to cluster with Alice and Bob + Cluster carol = Cluster.joinAwait(alice.address(), bob.address()); + + // Subscribe Carol to listen for incoming messages and print them to system out + carol.listen() + .map(msg -> "Carol received: " + msg.data()) + .subscribe(System.out::println); + + // Send from Carol greeting message to all other cluster members (which is Alice and Bob) + Message greetingMsg = Message.fromData("Greetings from Carol"); + carol.otherMembers().forEach(member -> carol.send(member, greetingMsg)); + + // Avoid exit main thread immediately ]:-> + Thread.sleep(1000); + } + +} diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000..8f09e789 --- /dev/null +++ b/pom.xml @@ -0,0 +1,370 @@ + + + 4.0.0 + + io.scalecube + scalecube-parent + 1.0.9-SNAPSHOT + pom + + ScaleCube + + ScaleCube is a lightweight decentralized cluster membership, failure detection, messaging and + gossip protocol library for the Java VM. + + 2015 + http://scalecube.io + + + + The Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + + + + + 3.0.3 + + + + https://github.com/scalecube/scalecube + scm:git:git@github.com:scalecube/scalecube.git + scm:git:git@github.com:scalecube/scalecube.git + + HEAD + + + + + scalecube.io + The ScaleCube Project Contributors + info@scalecube.io + The ScaleCube Project + http://scalecube.io/ + + + + + + ossrh + https://oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + + -Xms1024m -Xmx2048m + UTF-8 + UTF-8 + + 1.7.7 + 1.3.0 + 19.0 + 2.0.3 + 4.1.21.Final + 2.3.0 + 1.6.0 + 3.4.0 + 3.1.2 + 2.9.0 + + true + + + + transport + cluster + testlib + benchmark + examples + utils + stresstests + + + + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + + io.reactivex + rxjava + ${rxjava.version} + + + + + com.google.code.findbugs + jsr305 + ${findbugs.version} + + + com.google.guava + guava + ${guava.version} + + + + + io.protostuff + protostuff-api + ${protostuff.version} + + + io.protostuff + protostuff-runtime + ${protostuff.version} + + + io.protostuff + protostuff-core + ${protostuff.version} + + + io.protostuff + protostuff-runtime-registry + ${protostuff.version} + + + io.protostuff + protostuff-collectionschema + ${protostuff.version} + + + com.google.protobuf + protobuf-java + ${protobuf.version} + + + + + io.netty + netty-buffer + ${netty.version} + + + io.netty + netty-codec + ${netty.version} + + + io.netty + netty-codec-http + ${netty.version} + + + io.netty + netty-common + ${netty.version} + + + io.netty + netty-handler + ${netty.version} + + + io.netty + netty-transport + ${netty.version} + + + io.netty + netty-transport-native-epoll + linux-x86_64 + ${netty.version} + + + + + io.dropwizard.metrics + metrics-core + ${metrics.version} + + + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + + io.scalecube + socketio + ${socketio.version} + + + + + + + + org.apache.maven.plugins + maven-release-plugin + 2.5 + + true + false + release + v@{project.version} + deploy + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.7 + true + + ossrh + https://oss.sonatype.org/ + true + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.8 + 1.8 + + -proc:none + -Xlint:unchecked + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.18.1 + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.17 + + + checkstyle + verify + + checkstyle.xml + UTF-8 + true + warning + true + 0 + checkstyle-suppression.xml + checkstyle.suppressions.file + + + check + + + + + + + org.jacoco + jacoco-maven-plugin + 0.7.7.201606060606 + + + + prepare-agent + + + + report + test + + report + + + + + + + + + + + release + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.6 + + + sign-artifacts + verify + + sign + + + + + ${env.GPG_KEY} + ${env.GPG_KEY} + + + + org.apache.maven.plugins + maven-source-plugin + 3.0.1 + + + attach-sources + post-integration-test + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.0.0 + + + attach-javadocs + post-integration-test + + jar + + + + -Xdoclint:none + + + + + + + + + stresstests + + false + + + + + diff --git a/src/main/scripts/cd/before-deploy.sh b/src/main/scripts/cd/before-deploy.sh new file mode 100644 index 00000000..e2f794ed --- /dev/null +++ b/src/main/scripts/cd/before-deploy.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +echo Running $0 +echo *-*-*-*-*-*-*-*-*-*-*-*-*-* + +function decryptsecrets { + echo decrypting secrets + echo *-*-*-*-*-*-*-*-*-*-*-* + pushd src/main/scripts/cd + mkdir ~/tmp + openssl aes-256-cbc -K $encrypted_d19fb18b4b9d_key -iv $encrypted_d19fb18b4b9d_iv -in secrets.tar.enc -out ~/tmp/secrets.tar -d + md5sum secrets.tar + tar -xvf ~/tmp/secrets.tar -C ~/.ssh + shred -z -u ~/tmp/secrets.tar + popd +} + +function importpgp { + echo importing pgp secret + echo *-*-*-*-*-*-*-*-*-*-*-* + gpg --fast-import ~/.ssh/codesigning.asc + shred -z -u ~/.ssh/codesigning.asc +} + +function setupssh { + echo importing ssh secret + echo *-*-*-*-*-*-*-*-*-*-*-* + chmod 400 ~/.ssh/id_rsa + touch ~/.ssh/config + + echo "Host github.com" >> $HOME/.ssh/config + echo " IdentityFile $HOME/.ssh/id_rsa" >> $HOME/.ssh/config + echo " StrictHostKeyChecking no" >> $HOME/.ssh/config + + eval "$(ssh-agent -s)" + ssh-add ~/.ssh/id_rsa + ssh -T git@github.com | true +} + +function setupgit { + echo setting git up + echo *-*-*-*-*-*-*-*-*-*-*-* + git remote set-url origin git@github.com:$TRAVIS_REPO_SLUG.git + git config --global user.email "io.scalecube.ci@gmail.com" + git config --global user.name "io-scalecube-ci" + git checkout $TRAVIS_BRANCH + git reset --hard $TRAVIS_BRANCH +} + +function deployment { + if [ "$TRAVIS_PULL_REQUEST" == 'false' ] && [ "$TRAVIS_BRANCH" = 'master' ] || [ "$TRAVIS_BRANCH" = 'develop' ]; then + echo deployment + echo *-*-*-*-*-*-*-*-*-*-*-* + decryptsecrets + importpgp + setupssh + setupgit + fi +} + +deployment + diff --git a/src/main/scripts/cd/codesigning.asc.enc b/src/main/scripts/cd/codesigning.asc.enc new file mode 100644 index 0000000000000000000000000000000000000000..4a52ed30f021d153408da1a6c3d1c51b86936614 GIT binary patch literal 9584 zcmV-$C6C%2VPT1S;JEOyI|@Xy*pU_CLAvAXN4GU7XeNcZ#2Wm^A~FDT+G*p1PdS() zka6|vLd}iLaSOS_(Tb9GrLS=7(Z3_!Q?cyxslIXs(~%yY6$1 zX`EsFbM4 zn{VghC{aPig-lyjU6cn8L7+yZMj=yBJ*VC2%qxU51@^((#isg*n;U`rmyJ*rLwzFc z25@t}eMwA~R<`9}qc8JUVRjmx+r-O7+G`M!P&%r9fT5GH#EDV=wSfB*!neOUu!d{K z1L-sE=5%GoO`Sc*Rus;v+Qe(_VM+7K2`)0F=%ql9o6t}lYYcTHyWab#qfr^Db0hA!3#e-z4j&CuCva5`z%O4IbkY!r^439A&4nmg1jf zz)5PTg*mB!1mICD=11w%qO~G4=%>J6u1x{3!{MUa*^qXrG64}sgbQ09ND_$bDNzzM zugSWi(<)qdT=6^DzzcMv@(A#broD!yUb>R?QPyUHb>~BtQhhIqkc6YvlV2WmIBoM4 zML!>6L2LK1%-F7tU<6%IIRmnn*lq7>nW4oy6d)az$(>x1FS(gLrpnHWu)xrtOv=hJs3h|PcXjaT z_WeE+^Q%m*;&@-g9@uoyxVzF5;sp5iC!y1Hq!c!gIkaHPbO3pE<2@A(%Zuq4R$?jA^8`nC?v_l@QbcQ?Nf=N0n8rkqXA|Q}ekN|8-HZ<~`dz$6JQ(pjAZVP7ytg2% z5X@%H=06z2FyZF>^fNH~K{Xk@%HbFDv|fH~45!XkxpOf4F{!-MR)jpdSvMnigVCD#;_S&(|nKjD5xv)JzBC z6f6P0VlO98Dc9^2myVr7#i0$rg)+Vy4ZIvI9rocB0$n#DHb&}rm67tg|JU}lWEDXH z617){f-+MkGRA=qOdrHaaZhUf0O>SZ(D&wnaK>kn5Vv7X%C9B3^3-{8EJA+UIm75P zGsMuLH!44O;fYbOCwZG{arY_oh{O5nX9&#LBXn}pbwus-Mqxzu0FCDKX`=KxbH0>w zLVY|ed+~bwLe*uqw$B1J6EG&LkREVI{pt6asI2K#6LS0qJ{gl4bvR7PYRb*M>m$V8 z#8ov6n15+yq20TE_0-nno-k?uthplZcf3J+VaUo0JaPDq;yS$rskqxx9T5%iD!@h> z!Pe&VGZEdH@QlW#wU{UagAGjbm5SaH6{FLypv*1hQEAhxQ^bmMHl$z+#2i`SK@p=8 z0ORc1n;$lA{)M1y)i{bZWMhIVO#02om^GKWHoSv0YGXFS$TZLp9y{aD(`k<5Xu2Ci zTjPG<{TN2_cNU;JU!?JsBAJGSx&YXG_8&mn$WLVx0FDCJ`@na;^oCkMCl14}L;qz! zHj91qtlBitQGO1|!6i@B=?jX(m1aAX6vc|TTLF#szKBCT_P=iQ(JJ1A9*Dic1%sqXM=~5> zU>0}z42k&|!D0IC)}>6kF;{Avy{NI%V{D&)OP5{OW!hb{nQ`;1C@bf zGGSoVe-!a8zWQXbjD9nc<$t{cB;m7S>MY0;u$!oW3A`y>x<u3D&z6r@C86Z#U4kng6ntG_>H}ZxCPEeyL;5Bk|)BRXHb`>Sq zsohwXS{H|9MNzdq4s;S|4QfPrP-!@53fG%=d4;Db!#2Vi?Lg4MQvmEPK35RO+CPaI zDdi5cqh(BNyrHUNw^h^%aWS_G3upRc7bYHG9p;E0{xwjpTf-gb$t~+Kw3+%T$L?)s z{xcYFWpLPfSE9*XPE*H&&2lxPgAJ(V%JY?J9URyDXxQj4O z5is77>99dQoPRJCUb}S=Jx$k_=B=MrRi$3BlAq$pqC#eI(aWc+O zcFRf6SiOcH#L;l>&vLo7aKqc=82yKW1g(R>T(cc6Uddu)=hbQSFCQx2@>1~?<<{zLM@(@ zD%>{Ep2WbGy*L&ue;B3gQKo#_$mH4U5V=e}8BA!Uz|$YBo{W`&21j)CZj}ynbqr&S ztL*xylZFXIKQ?wia(N^8Go&yEuti0cBr9z0h2>4 z{*4d;O8S;@;VKGKm{)rIkp^dQugCe24)@MoYI-~qY#=_n&vNikx-Wz5e7MxF^UP8L zDaE96n__yf$xZs-U*$h-lt_I-pdC#N9QzjvWPpOXoxh`VVgg#Dn2?Z~8?n=N$9|h< z2ALCO^`=>6bEMKS@tIKFgpH`_G^U%EjXkd+jT=U2?nFKWGt=_|qM&+?64*S0@84*K zOcbD^I1ZpeT_*PBqXtigw$zAG=#%vGMhz2pKq0SnlX@K%gXO^vbO`!6sxQaTRYraz z0yScM9XX4-1KMgrLDE?_N0x{cbW|$RB;zD5B+;0u<-S=?B&qsuix-_uozXyxE@fDF zeefsjBB{=~&u0E+C?W6@)nm3ZRv|S2hg+2^fkz8ysw-ecbOff7 zoudRu|FZDo9r0A7H!4`sJtrtlugzk+tgcxup6@k<0yA|ByxEX|?T4HYFlVZU6vI9> zYo5}0#{o+p@rI;pF$l*3Y?G;$VmX-93_>)p8%LnvQ6am14K|@A?V>ZEL3mSJ+F*Lh z?dg(4#EQ%N>91fbZ1d2fr)lp@A9_l4*C(`MAYraJ?(M=PuDURDj9CzKDmF2-6gYkOJ%0l zG@;HIkz3-lzto0*{^(!Z?0{JV@-Gn&us` z4SRhvd`1t#-z{1I%R=iHATT5md0C_^U$DZVTK*gbkHHlazkM0)Npxzx)5LGxsShum z+6+uHl*Xr3l3pW)5hj~t4U5C$jr?FM6vex&Ww;(=@yHFO^=v`sPt4eaz){L@CLwv5 zyz6kb-zSeg1-I*)!VZDIfK!2eg0oE|4rkZ3uBo*wA)Wd3Ej*)cp-W!!FTFKWtu=pu zyAY=~VDZ3L!4`1xP*Z;4;iDRihAbboj&VU<#7Nm1AL^t-*uAG^g5qtrk^B7Z@Wnqw z9Wl_l=2h%CaEP?R-&Rr^$iBa;;5LrxU(p%+55x6Vs;Bz6G6KV6CylMpTBEw}4$`sR zbnt6eZRzVCOJy)$qKz-H$(y;0QCHJTypXOxX37icx2z;$dRW9AdM4oOjqpcDv76;A z-EYpd>p@Ri(VD+yb4I+u|K>}m29Jh-A1~_ksaozVYS|jpsq~?)k1HN))DSssA5sknU9v`j!g^#^S{cK@=VKPG8P~ zx5YIxsK;7}6n2m$$jE}uC(w2mpD#r5Yjfl5gaQ>&|1>NN>!E%+9F6GW$nA%NnzIiQ z!hV1v$~?N~_!!6addKW_BZ3k5y!MCanj{mS=bXfDlvE$13#V5s zpjoyY4ubB_f$jFyEDKt;n~&I~Q1Z2lUzrw*wkT>QwiFpbRLBG1dV-n1#}U(mMvcBG z`)OMau&oJEUWuv_z~qdyd+vpB@9f_o?bX^VZ70TLsIR!Kix_B6Ri!qi*%-WQUYwCJ zj^y}URTLw(1{(T@?RMUh(n-Ly18e@~j7ZNr%kZDJnf}ni&!lEmg_T}cc{6}*0IW#I`W5;=IV?v6zm}$TuGgAMo zV2=>gW|Fo{)Md9u5(B+^*tlQRL*7}xkc(pLk#E**n8_5zMIO)8$>AalfwF&I@*NTz zW0y2NSEU3PpMj@Kd2Z6@Go}-&;1oJyTgV)~dipmq=vMe?Lq|ZGoN$aJJx2dbvdsmo z(9jk|6Ql*PlX(l|QNQWeg*|ZZIydX<|F41_^1>p&UKYA-?R4C2B~QEU4wS$nQUi6y z`}vJAHSZYpAF&6Bu~-Fn$Y8T`rQd4OPaNJVnl`wWxguEE=Kom{F?hmeOH95l2Jej{ z2?_ul)US1?Rp(HEks|{fWt@77^qH$3F7mmMY;215!` zyI7ui(+Az*_csE0Ez>d_CaWv=U9~Wl99_O*aS>w4{4BOsVg=JtSw6Y^4iy3DWW)M> z)jwkdhGVB7Evhi8#9v#I%{(7UI=O$8Nb386g6)IVV`*#pSzn)e8o7Cs*K1~@ zCR8C@1zZ$gAIe3^MvE|?kdeh=*C95%Hbjz?uD{~C$>WlC&M~qSUkkX?S!VnBamVly z3kPccc15ozNywrOGW8V0?e2adU9*T*UquLlar_^mVV^^XK9-Q;^o#L1+TZ>` z@|9Pk+b)&L$e=Wne(WP5*%Mo=$tpGDw-)r% zZz8Le&82-Z9;a)2nRWu=?OD!QP$bE`@PspErw>;NEejA|@t(Op3@HcT1QMqmHXS?1 zYrP78kqJLgo=)mRVbF;`rAk#QDR!MXw4_gY{7LIO(!GoPhIL;&s`4t;U`yB{frZ^Z z)flFU<~9PXkBb|`Rl;Muv=e*X&+&PX^$9*c1CLuAfW`VC5%$M~3AAzhnsM^9#r!(K zuc(nuIZ5yen7?A<5E&)IH)M#=1}#6oM_$qhnx98aVO5JfS1+_fla00hsidsz*$KpV6wEL{^l_R z>+JAf*t{PL(%hbGg=G_mn<4jwK&WDQ@0!9CV?8E&`GaoAwfl=)2uJw~V)4D@8S&on z+CDr|$&e^anFj{p#20c{cXM)b{`0Q!tfRMC2P|uu3g94Y*e4*K-*!#JvsbLhf`Rqi z7Ld6Mq}7T-Odg6#Op6AX3xCiBg=;a5R4*VTkIrWh)`0>fGyaUjW%d>=G@rHoz%=GV zU0DWB_igC!{&ruWUUTZ$XQDj!>le(?TlE;#KNo6cMCL|sfxa`UklUcjDZ$irFgSE~ zNtBrnpvV}K(&M+6f(0Ej1~oHBpplRg#gJ+zz-U?m&zC+^q=|$(hJtY)phDFUh zADwg!j_K#l$k$A!@eY06_uC551s%1=}a6q*s$+OT(o1ru+JO5`(F zPryCg28&$C!V@Ty)#aSKOV+qP(FRsPQBa0f0j38&6BXjOoc-RKNBUGD1(_-*c2GTw zqAZ3t)_hJ8CH#raiL$yROFZwW6cD$UW>Lg?k74TU)U%kqEk}CSDwe5 z#9{8z$5(P?m1a+OL_-MXa;AhwsZ^#eONP?dsyLDv8n@u+eL0DM%K(y7A4vg%BD&8< zw!e!cYm)_MgUF~vz;08;RQ7P~2bi@F{_PCMp4!t`*%6O%;lzAYo}pgsZl9@Nqb3AnDOu1Ab)o`?cRVWTU{;f@0A2X#8Z|o zg;$O;VR_$a0jV@UxXS92*w$hvM$(}wvm8Y?qn~eBiYc6#?NSXfPQ2{E61(~Ce1#Vr z@kQm?vXbAfVPh6QqR$u@G(iwcqyl}328Fzs1JeES+aLt5p?40|+qU6BZK`06Ce@_h z6h^zm+&hUTOBv$k4PmT<^a)~s&xKDL>dKniI)HdvTblG#_F;MCEt6sOIA0D_`y}@4 z$7~Yy?K??n)_2|^m!_WAAzH9Ekc9=_)7$6iuhZn%PF^#d8%+^sQa$mNZqGEwUFcnY zpBXY7lfpb4HFhO^0jNSyho7FxP&dw^E9ID1k>JY7L|Zn7#8yCcIEbz99^kX8$oDA$ z$gg=J%R~j6ha8$q<7jajoEOqnZZfmP9e|?Tp;hE+CEA$?TVm#Bb#+0bsCx zF$+#=)xwHOv`apzt1og>#gss8H!P%i6Jz3=g)7c{WvUE3#-9wbZ?C=I#_0}4awJDkVo97Vwx*r}@Ompz%iEP&usk0|1BKl2X z=Ww4E@61>8dz9LmJ)egKfleNu_8OdBodr={oY5@@Afg41W{MT_JDm-W2G7OJQTIJi zBk^7t(%ahF*Zt7{nmEqX`NO=2@I6BbSGN!W&P497|K~a)Fa_i_{fF#WsOKomf3q?} z9AsaoQA5uWB8vuHDg)}t_xDd6$SIz=j-Z_ly(t2FUyz6%a;i{N(U_%zFZ^OOgq~AN zxRpQTG3Y&N#qehqaAgu`Or=H$mjS3lXKI*xgHEqAH#dxDqY8@y8@cu2E)6lEkgcZW z^*S{ew`cHsm*!NaB={PeCjbf{j=5V~u5cD6fY9lZyTJ_EkG{s^2kHpxMw{3`b})DI zEMhw5MMW@C`GiCoVzlHVC`(P^<=hPf(O>!@&@4@ssPD%C4)K5Nfv%rpsl-sN5=1If ze^Xe~{o+Q@F%7jGcDM{|&EqNN&#W8a?;sfnf=WixBxLYE2}j+wr#`RGM3jl$g_(=r z|Fr6y#pXt}{>_?M(Ee{a6Mn#H<4tPS?CBm){wVS57UIko3%_58Lv6$fv5AJWLq5&J zUumbGtfuN%#V`2>LL)iu3pP=-=y#kK2+=N01 zRAnQ;GQ+WpkAo6EkUNZZ|INKDc+~Qwu9y}ck$G%G?ii!)qxfw#N5=yy4W+xnro#~c zb0+ZHR(|$J7Z^0M{*Kecl`+QK{@oa>38^k_ zRaimQc@C+(zk-@*1?Dfzq75=np1}SG5wU?+Cybs?J)&fNh({=X0;_<0@qq*3Y`pwF zwyoRG1-|lehVfSJyZEbEPuL;*rM5E^dujuIw9Y9RSsv78aXZr3o;#w)QuLM_r&J16v`T+%!=;2X7dg95hdov zu`aie8I0S0vt+H;U)DLlc<~LY+E@H8Oq2RDBhLl4%b`wbh+cS@!emJtz#0f=g%hSr zxGbr`%2(^9x34G-|0!}_!Mvw7;T{T1r1dqT)nK@j^Jv178WxH@dr)2FrSHyzuD5g= zz9IRJRd4WDxX`SYHNV5qK5F#O-sa+MDkK~PaahfWNJ^hvGQ+h0*QEuRy1VGq0X_7_ z)}F33;sk6mQKoszA7b>*Tit%obir?oh266?Ddn9Kd!~_H&bpLGeqp%*NbJ#u3i6Qy z+#bPSQ2!`l=A`MpZ{jzNcJx->;Fmwj!R+%MH&L)ibcvA}m0=of)@%m=#K0NLp5zlR zdVsb|(RPT;Z8(W4U^?m|(Sb)d$xoFh1^m6dT4jqvBQ4H;@#fj7-l{?q+&^~#I%W35 z5K2Uwtk4-iDjHh}G4P8yDU*wn8uVdG-20Hob<7*Y{lZG7sh)uL$`Z7X|N5Q+GH$wT zMq?@BI{rT%Nl&5%{GCTa^-pbkW#(V}(`Pe^O?7(1Gh9VuGYnfNavVC1tQI9y^7xN1 zrSZXs?%2d3Fy6S<1me~6Tk^^LgZLz57i+q7sjH?TC@cWEfBS-u2aNDy8N!C?!cAAg7ZR8Z*5hU<~q9U+XqiX=)WcN z@akF<`Q&Uycfd;WcE8iHWwCs5E(HRJA$e{?#>EVb$t6M$Ry~co0`Vqn`S8z#?+f+G zHR6J*?E!GW4`k8L!~x-b=S6Vdw!Y``Q(}Ep`BfaNo#va5c%iIqpFX%axuo}kidxyV zK=8K7A*^gMD(4HO2C0plM5NWe#T*!b;~4pv2*FuNi;oi|5sgq8Sp~>%Pr;$HqbUdR zk3}>nvq|(D85M6Wnvk4w5YcS-j)8rM&J#}MyK>J)gwH0~`Lw2JQ0en%)qAuKPhV_7 zHc}hk=Jyw5$otYZc3gJ||MI|^5BHf~e;`+*58v_%_{5ATqR?q0by>7%ok|@aF7QdI z(WHBlNrnehu|2D}wZ5!%GD{x<;Iyo;Gt5KlWQ=);HW$kKPEH08YY}Wm9i9v;HXgcW zp>=$yB|rHP{DZ##9VPjR1N3pN@^i1y zrMDfdS`?W{Qb>SUj)F;)m*G>e{}ig$@Qe)s-Msmy)-iQuN7f9Usn_|eNn;CF+QrY{ zKae21rCA5R9n?Ixj&Fwx4|4P_jjhJRJ5FrpKne-UVXOS--4vlgXV4!Xs)JE{62pO< z+q6V~9tbvb7M@krg%j7L-ZiDXK`wozjB&Y19?Vc`KzpCAQqw@N0t8u_6;aurDWI2y z0Lu?ouxGTRd&>d~9A(<;0v z_%8A?-KoM_*$cNhoY&B>+)cW5Y(TDrh5-rf1V9hJh$wU^fwjIL1j~%OFDO-k9hLb) z$aUIouI*Dq82gylSx=%f`Ty5-x!=YZC$`MwS+?oBs4$&s)@t~-CR^IvN)|qEolo;q zrh`l$9eod%ztb!DhD5kINELDQ2B}if^s`jrrOEurgT;O3wTLsXxe7 zBjZSMI2?IA`z2XPxzl&%pRnBm6Hue5QoeGRgNd2_4+^YAyE%{ga7GDB zH?>kUdA^_ekPz$P@M)n4VZ7s2296@|>si8@DC2I~Y1>br7rlXXv#6<&7xpHHn82JT zpXkl4yv>-SwF3>n%UsWEQ=coi#RQlP8ZAA z-iO4HpFx%A%5`v*4~m2q6`m)s3dDjW!)K_Vp0-UhbR%*uX%dJuGbGFxn%|BpS39VdF(7irJxA@?|IGPc;3 zUWr1gCT4-Xcp~X(c9BZc^T7--0dZ#ex#>B10}f5HpjDULBc8sqbYKMohiK?ctt{qM z8M%YB3$IjuZ`PR}?0F?En*}V7?PYr4Uxf$k5|>bDG7yMH!SyONw8~VyU|XZ`eZ3cy z>yWsp^>)RYFbbiBWmGF0m6iri2!iShayQX7MYitbavFsC>5)qAAu-}02zj_fVsNOR zRQjv;F1%o62;f8-9-Sxd3HOYI)^J<1Or>{q&RZP0pOlqdnEhBuC znAvxBI>iTwq8F5=pu<_a3BR~loVB51vX@825juq{D`GIg#F7d*cN_nSvW+dh_1_8& z&2tc<1w)jJNjSmM7HCndoT~GTL?W>darD%r7D{NTCpRn4py*{pj-*M;oS9ZF90rX! zv*F{f3+n78i~%av#IXYc57#XWX~%*?7vhAk$xRdpE*aZ4+m43rQPf9b)#6mKxwoc0 a*k|QVk=1pNdbgnI;|HrJ@k^s3wEm;rl%;L} literal 0 HcmV?d00001 diff --git a/src/main/scripts/cd/deploy.sh b/src/main/scripts/cd/deploy.sh new file mode 100644 index 00000000..fad5cfa5 --- /dev/null +++ b/src/main/scripts/cd/deploy.sh @@ -0,0 +1,9 @@ +#!/bin/bash + + +mvn -P release release:prepare release:perform -DautoVersionSubmodules=true -DscmCommentPrefix="$TRAVIS_COMMIT_MESSAGE [skip ci]" -DskipTests=true -B -V -s travis-settings.xml + +git checkout develop +git merge --ff-only master && git push origin develop +POST /repos/:owner/:repo/pulls +curl -d '{"title": "Prepare new release","head": "develop","base": "master"}' https://api.github.com/repos/scalecube/scalecube/pulls?access_token=$GITTOKEN \ No newline at end of file diff --git a/src/main/scripts/cd/secrets.tar.enc b/src/main/scripts/cd/secrets.tar.enc new file mode 100644 index 0000000000000000000000000000000000000000..ebcec4f6d52ed10de6e90d1bb662d14c697e95ac GIT binary patch literal 20496 zcmV(hK={9@39UuMA0y1TT>754;`biT&(F17Z{9*2!Myn z;b>v+AbcDWpDQ)@wJUnYe8rsTRLS!9^H6ddBVJr{k1pgJdTN$LR$-hqVP1vbxLdag zMk5iQl*Q4qRgMu{yaFoT5sK%r3(?*b_cgD$Y70}ZrEn-1ELD@9jvtV)^T)Dqu63Yi zW7QOtz(8K$(m9Gj0cET^C0FH-|Di;X^ywg43j`JezL<{!tI80WL?|caniJ8!p~tVhScAX+~H?}hwBfQ%u4+5U18{JulcWsR~loz#l{l>_5o znLSNE4cF=94lHEVoC*Ccu)0tMswX#3q*-$;9E67JRA}$Dq*zkVqn8Uzw^BI3~riey`Jkyq9F(s`vDbDb}y*BYDYYizOD%6X_7e zrrYT0H{?7je5%XK*ea~_+ZPt*ZC*<0TLdBQvwO;2fUA6s-E%!K+=chI7r;_=JyF%! z<-0>2Y`vPNIRWH9UXcT}XkfxaIknOg0F7gWDc5AK$P1<07Rc-WhUnSxlgSiL|8mg9 z+ZlIty?=a9c05ZVSi6@M#dV8rv`Gig=I7tos}{4$e$hz3Kpop=!DbNi%0J<_%2Gb; zbSvc@pe>U!>c+?*27y$?G0G3qMdSlr|1DN(N>5+eH@mNS|J#V_3K)dmD9#iQduk$t}K3 zXF7o+ijwTk2CQj**6ZqMBz?&*9)OLmVI2LVAEpH5RLuYdSap*T+YBswOxI${puiuN zQ!1_Kyeh!J!|VA|+I!04G;#a9us2^hb6w!GMQxxp&jv5l17CqKFuXb+u2Soa{kcf= z$vA33{@0!TLv}u4F|yKaVXxWyhY2OOwi)0x97ib5dk<^I^=T z-ybV!8h@S$nyEx~dG{J~%UuJn7lV?m4;=T@j)wy7=u>2*CC8ZO_zivqla{f?7BOzLA?8gQzKsdpw2}1l^E6x>u^>D?hg=Ki^ zv*bBwM_q}Ug(e{@|2QWRaWG}ITIeRK3hyHB(P!E7J5S-ZI^?eK&Gac&=2Ql4A9B$7c5mA{8nwYwb6)@wN-Il{&myhm;^Z`Hb@Z`2*w5 zBVk{FWq@qO4mXfoF>S+W5$07&a2dx4!5#nCt2s^_R_ zb7lGANN-crthpQEYAkcbk9$}WH(AFv+(Nq&3QSRR@N(Q%#^KEoIv#=X+A7gXw`2P| zCin#ErVE~lAXqlzxQ&)Br&SPK9uS&d$gPIwtK^B+K_3Hn=$c=}(1TLHccRGk@5xS_Cm)K4q2La0o!ced+F^MV^e zKq*ZoI>iQ5!gW$(AMXXznNX7(e2ATZs>kOJy=MRJS4@Z*PZ|aD^2F#2T5rP`lNq|D zroFP~1`~aKi$tTi0n(}DTpK#!H}K0fw0nICa1en^GtUcK?$Xb{j4E#yJ+>o|O$uTa&(r;kEHBRDUkM6Jif`8M;G#hO(2m_5( z<0OE>uC@VqyobJ}shBI5*gXA~lVF|p2FM6-cJU7CWAKn<5cTJA_Zma&3I;inV2PW- zY?gi~rlT#393M|+rj0Iw9Ol4MG7J`EigU1*A*XBt` zkcx?q;&u=qYeb|E@u&*0zuR(USdYxs3!-=fR2s=~C9fFc(7-bcq*apZup+FJuHiF6 z&;O0+H@EALY_gZQTE$xsTBKD1EME3#RVrsLVy#GY9hr#+YviS6^+q3|hKzLn zuVcH*J$3!rlQzekUU+zzNpO2Y?lS$_y6KwUe+2}9x5a(1AIj}~xt{NhlG2riFxt3y z5TE=ODEHW5KFQW?#eYs~`%CPD|OvHutspOI&}uCmJ1S z(M{vdhU%HZJ6kQD`1KKT}CuZ{<@T2wGAJ=t^pG@ z;AE>!oSK^@8gm{+I}5ONK#t=94J%@;g3jC2ZaFC&C1rK#idK&ZR(lX=3u7ftX7p$U zZYRe%R_fC7(()L!N_Yi(&?2*&T72H6*O)f#D~9E;Q%I<7=`8au8s1^?rwiZ4G)^W@{`lT@vCVqMY2wliVm{c*<*rmrs+UyrS!ZQ3iO;!>G&g5=& z8Bi*FHDh5fchhS4O12zWUA#fhA%l|T(rY@h)cvuqFtb6K^iGM7Ia_uA7^u={t9h~Q z(-}qX*EJdRd+g10iCQ?<#tAWL(eU8Y|6!IB>ID#Dnffe7f;mgdMlf@!b^x840GUQw z7^~q-YC+LhH7cXR<8&0s7f$ z>cjztA%sRo0p?#DCcic84M;;QY}_4QP&knmtmxoj$2ySX0U^%=n=MG-#@A7rA*w9E z9virxvPD77GsT)?dvngGumHjTH+#4XNdIw`FAOW6!attqILU)A=~?%6c2=W8ZzAhN zTsE~zlMJh)`=b{}f+EjQdG1YkUHUe2PT%Vn>g|KwFe22tyqiCQmiOrIQ%j+bmAv<3 zfN}RF;ZLKej5Cs_I`r&!)r?OORWnhR4?)-Ufb&EQUd6l}c=Mo!2(W_Zv8N4BL+xZI zT6M?qF?+g=#REp#X9jp%g1@9}RJFm^w6E0NFht_zO*S_0V5*HMfPCk1?z_f6Vpp-v zmEwvZB2KsTDhr)Nn2s$7>-KJ^*)bgz7;n#iK)aCZKR`6LHWkRd1tee0B9W}2CD%EY z!}EonCN*c}Nj}_lNjcZg&ASSE^bXR6qQ4QBt9Tu)dtdTP+Yb&C(aM%t(PY^V-|7=GOZO)jrbSqC5(BDJG z@UCxSAl(oe&m1>KmcbqC=V}ARr|kOpHuFK>tlZnZv!}7#?4vFwW)O=vD6o@up?RyE z-^X*Wb(>M;{ZNYLln6BN)g&~M90-Bp%CXCpx6j3BfjQTGt-Mo^6}+}ivWx39vwTu5 ztE{DGXu6H8&ymJMQ@aX=1MRo8rBNiVtj^Akchn*LI({$iZwX{fk1xMD;GUsKh2kgu zAXEdotW3mYadvlO2;+xomkHw2tqP}lvWAY|%;lyW9TuO(O`RQKT!39kf$AtCa4m6? zE3>3G=`2cc#C3j=FfW22o+$9E5?|i7bWU&$#IX?p!g=xCN;mLe5n95bW%AZgI;3T9 zT`;lm-7c4;UW92D8JYt5x+J)o_kDA~55jgO#*Y5Cwqj#ig-)Ld^jm}l#6TRY+WD;& zO>+O;owfgRFKpfF7kjvO4b5)C(+MRc?3FusFb;|llZc5N1+BF*xQ=UbAI`Vuwk@9* z!CVlf?-`(8!|b(&Qnp~@{eu_G_PU9*f_yj}^ZBV7-~-;Q`@b-L4N9m^V{<{rD=F;# zwe9&J7#t~`;s{42g^{=+Y4B=ua35P&Sn}(4v2C#wlPDHqfX59-gLbpeQlOb$t_(`3|tMp8D44Hs+fj z>2wb-ueW47M|ENP=nm3TV|Bt#N9!4H+&n^NOQfMyy+JnAOwd;Oqk$`!k!KMRoW35& zHqCj-xXgZcz5aclb{z;qy?HH!#+c$+Nj5AP0>7=3Im(?h*IBkdFu4#ZPxl3B>~`Nq zFfkSGDdY6wKM4S<3j|rN)-x{V#Id;hecKwUy{FbXnSR+gpeGfD1@P4VQ5^_gpwtDx zOKevyQzcWdOxOWEd#T1GwL0i|Qb8J`mJk8vugBX)U0}P1k#q8JTI~p?X>TWF8j*+x zGseP=G}LK4zOU6QqY?PlYaNE2JnkNgw6AzXRK4`feJu8Ao>$kQIL^m3 z&)t6&Q>6AF;WVTcYf7@0U%<0fe8x>Aq)S z6&Y3uv73=L9>z&GYARmJyQmD{qLi=mD z(ocpC>*jX0TVwX~?jprJEkIb2@~>AQ@_Z}~#`$LV>lb^s5s2`-TV_`I{@U_tFaF^(QO+O_5telf1Uy!$J;BBycDM=DH)&Q? zBPyOzjk*3%MbAOMZ4&ZOB0J&$|E>S2{WpLzQcCg%p|gHw1eHb4P3_up#u>KmLsl}= z>YU`AD6M*k@;dMa!3IYjm)%eDhLyGjb{f%#p$F5X4~_FaeqSEwEv?^I%uaUR&k8bx zd2hATxaq&7{54RQ?RjaOA)`tV1UQ8A)qGxN^7YL(L)}G&3Mf?+0qqKXL_N3-irsFC zYLJi>(#?C`pu>Rw`(v_v{GQZyu!}SPwt&AYw}2rb8<={HXD;TI49l(txxoqn6Zn!S`TvQv*)z3Y}-4g27?;6j8h@EZX=#AiGZ|Nj6U|CTg<#EEQeV3N9Ua#A*aZ zCA~|hO!|O;%A(H(U$>ggNl)qeuS!p!r37XsSu1YzvL9gt|jvJ6?qDp$5VkvI=VityLB-D}-p49C*O zx_`r-2PlDzBi?VzK54^nM9b9rv9rEk;N>|3km#XD&R!Fcxc4f;R z2H%TtJP{S+Ktm~y*zE-tteH8ug_;p8TSz3QrOk20nWRmo3}Y?luP>}=^l5f4V|uW$ zTl6<3@t=4=Jqqs!a4$vEexieN55&?xGsduv?)I+9-wQnZrpL&vEy(RLbz1p>uQuN} zr1US;s~FmuR+|&lksvGyed`$T7j39n1iAXbZ?cwcbr>ob2`Hn7SWA|m1AMFV_OC@D+$ISSi|){nL7)tj^S(4kWC^;xh|W06JgfC?)JKYlGX=v z&r!8z3hN;x_;ROP>OyBO9WREj6`glg>Vgs3kct?4NfnY4t%)cgu2FW|sG4n_afe&v zE@oT+pB0epQc|BL4}vNWZ5$aI%cj_{9m9Mq|>#>N0?q5>k`ZQZw(2FszR^9JW3<3(F||k;lV0% zF3h7!{K>fdiA)lf7UOTCXbWaed}EoQDP7fVsUW`vPGZvwB}D1mr8;!{ z0v5w(!6qd-9l3FT^7)a$hUPFHXzI_rXM4Eim;Yj8JK`uD-e2IItOue}sZ}+s0&H?? z%M&j2L@+YB53?KpwU{32U}?MdX}Hw!gA2?jOV84pVhjWnC-xH*S3V&~doUPSrEvf` zSBlG!NcJT9v33jKoYJs<_nA$(XkeDSLya>h0VbjlS~UhS(FZ5MTVV6|(1b{VGJW3j zuZuTpxiknCk@BLA9`r8AXroZ6n4hM-Y2O03<-0!>_sOmkb22XPZB?1&9;h5$NF>54K;FCU=%* zLbeKE`r;msl!lK~)){rZZdSla+0>xS86_fkjyj}6vMQE&+LcN&=Hp7Wz1elDzMJlN z)tQwA^#O+zwsRE-px-LW8_cK1TvsA=eizEwWxTg!1&&f_0|CE*3HNE2gTeNSI&Ok- zA?X4X!`7AGHu)f@fSy*M^AR0ZM4_sDiHk%-+rtTtVML-f=wA-~krwZC47R{@c%zcM zYGrnGZK9{HJh!Am^woll4*z+;+ntv8VAp{er1qRTSi4;20BtT z1R(D$BHQaSoc%}|ZWTn-ph_j`?||NQ--SZgR`Le zT^l(Bqa?p`{GAA7)}Qj}3Y?|k=@=*EKn!n?a?_mvhi$SVZM(5u_XyLGi%wK6ieioQ z1X4v_W$nZnomy81Im6G()gjEvcT{BH(U6Tt>y0?A_Ua1DL2LN2PqYL6ISRb3tAPij zh=}O05moNrE`mr>`EwA9DsyorW4BgobNu%R$hO+Y-|5Cg5);SU7$eP&P`id(@uDnM zNMU0#?&^wi|Km-Z!{DCx{YY?K%nEt3?n@ttHMcCY*Yh@~W@d;Py)!!if1Sw+M6v(U zSqO3|Wf3kUj;XPdTi7#nI9v@n7SV;vs}H;^hk=_Gv)IC%k!wDJ$ROK8>&;BLs>6Kh z8p_>-qc$F8v(RCow$ne_Zb?#4KTW=)j#}&fA6%-cup*>w0dM2ssN4W^B{mzhO*aC# zZ}c;Jq1ITR(x8C>!=9?lQj4tmLA1)Fyij8bdN!*Y(3sm8<{hM_URI2+SarfeFNUc2 z(&&l4&|h;J=wmH&BNKzjOY_B%Za+%wuHuUZv1!EdC3v9fAiZ+H%*-LA5u&SW5>%*k zl1lF{M!{yz>GWNmcjSf7PN z-M$rl!gmZu)B0n`cu;UI+2^-9Xhny)t7s@wYRfhY1-v>!8Bf6 z0aVuvqdd4kh)C5lSDf}aJZ_3Y-_FPc5LM2s3&`lB&8NE}S$u37xf+nZIE_L(rX!Kl zGw#!|3UDL!*+#U~VoOX)){dcQw??O>CU?2Z2&GLzZ!zr=aZjmLtdcnD zMv4Al?s-2#9{2_pbZe=I6O58%fs);Ck$85pd!xd8X3Z>@#!mXmSOp)uGCVf0o zL;8N>6qzymobVsFow|W&(go{C^E_oS0E?=~BQq&p$9soU>FeKsA(!pQ5Qg`jf>7Nt z&4>HSx%zMAxof%mIYp`FVX0NS?!BynITYvrzl~ z%-ZQxH(OM){m#XCbR!2#a@+r;xfyojl92&T@_;r$1UUeSnDZb`ss65Hrr|^H@@wvs z^)pSkI1~Rg*SobwdhQPU5Av5%y^<0SqeMw$_Ztu-7SLI(%o@U2R=U zr!KQ|HEcS!tK#uD$$R$h8#x_8DVYg8SP5P|QxCKN0g!9{1)HFlEt}-{L~HsU5J+Uq zVfmKs^AK!CpI52EJDZw0{xkESu}Mwk{ItMpZ#J@CEn#hkAfKgW=QFDBQI%L%(DcMCXXoM~&fsk2OYWL6-NQgC z3KnUntsVG*V$jrX{w!FwQ1!MvkX2L?GiT$JB;nh6TZR9d+m$`Ij?+QeWiS}vhVe1MeO8|uD&9O2exBg_N_(X0- zIV_G0EEKeVYhG4>S@_Hj4_wJAyUk>6NhCo9MNE8!DbVWdIe|hYXedcX&KHeDH6k>6 z%>jo2hE^i)PccsSA%)Ur>?v_5%RHIwM&q`gR$O>*((Z$w7>2K*;E#ZbY?Fy7-q@Hr zOOdwW%h-MPRndb&w}8!!?ylS}bO(O&{hn*`;iqUE1JSv0Mc@PrNvG~qK&zVjA$n|X zAF=Fdo{_m3WaH1-yQQoNU0o0nmc*YgGIIT2nnUMx#L~g#^7=Tedc}z@{4&U+eDktu zE$p@dPP(?c$?Q*}Q8=H6Ca;Vdk>ZiB&2;XQB8!M|3A}||nDi#E2k3Wdlq({)*|QoK zfZaMmSRtxmt_|hFz&#VtvSBlCF@C7zOiF54QN$st_23aq=XX-7E$;h1!r6LkXwHL{ zG+m$;=h1zY>Xpz?P1L9$NOa*sa|iX78A)p+f!4#(71Oqt_XL){D~7r-=CU$nvjZB3 zV=aGbt_>|t9~qL1xC=$>D7^G=oGehd*|sPl-VVEEpO8}Bw2kt)*_U8$?Ca1Adr4;K z8(^mNIv7h>?Tlx%ePiYej<5vok*MSKJ*avwvOGop9D~ChylAPu<5?(?Zge;* z52K}qhPP@Qfpp3wyZIo4vjbqQ+!jdiwgpD98RQkj1is264d2g|)wTiz7}WSY*6$~@Bw6tkV@ zLgU>f>Gj&i=JhOf1kVezypPsUw7x1a5cO^sG|8j5v3pv?&?S%b(%1b~3V$ZFiW+Gs z%og3?Ax0XQq-?*gBv1UEP6L|oaWH>CzPFk__-ljjwA$f_ZMpc1N4fJCb0+F7BM`Uv zpek?AQfZ0JH@Lr<%HF?tLpJ8rd>J(8fYd?Wv+eTrlrH!rc?8^q?XBwj3Vhx8r*IY> z3oSoJPZz3~Fu65tFjX8UC0$~=s0UrC3dgeGd2|M0G0fvA2@t}VG&<{(H=^+OT%;T$ z>66@NjYjx)IZb|vqEQc^0^h;(l`0zh)Oszrn2U0rvZICc)WkQ&$Pv~-h_l1N+cb+U)Lm<-ann0 zUGNV+8{Vy0@6Nnu?37&~N zp0bR8cyG<9TYq+~w%CCq7)P<4A@LaM$&*AnIH+j<&q<;UAdemBT%M4mGk3w~(@~Zz zC6ybpe6OeHQ5grnwmab_pJKE>xX~*rMJW*vqo?{BfBgdUWDY0ekWxy`6Buus%L|r6 z_s%eCHn4YC`u9LwM?83%CIa#-_wV7;LmpdMAasyhOiNhkuaRh27uT3b!ANF|!R!hD zAw|FcfS+3F}@0dI{A|2@u??3qSmtKM#TB)*YniF>=xjUX8sp= zPYG?j6@fG2a$pz{8VOAlqVL!PZE~9M2ZN($`&8YYnqb9@cf6wGQ$;iF*yU zF1a@C#5G=YdSY!*1F2`bmQG_z<0qyX_0lH0xT!zc2pYK|_L@JBA8p~mlPQgCHa)%dr%)80X zU0^4UDp+ljUA5F_qddr%0V{msTBvKfY(F-qxmxr0&jOg?wGve&sKoa>pY4^7Q^AqL z5LOy9+is?{nOrfO~D-=#4sa(ZH*WS)DHZgD70m;`N%p_E22j) zm^4ws7tynMm1V+$oX#?KJI2% z-aLbn1_YisZPi?ECAr?EB9)vN1jw9zV-o+^1ZU)g{6rnEV@2jrf46_7yApF1L?yYV zq|3j)afD1GD@RsBibr~DaKpV^Q#j#rr#eC=?iBL0;W9c}s<+@YbuGCBP+nJqqk3dW z0cDG1S*@-y!K!^8O!RreKDBdr(Z_;|nlTTut9t9_c2+!rZzYq4qbJorRJL=h4C61m zJc|*PBueG_I=ofjzw00+A6GiHNM76SOhfq=jA(J0DooDc?%j@hYvE`JvP8 zxUEs`eEO=2vekxtV$;+Vqd*~q8oZzvtSGZ0{2(Y$DI$HFE^muYc>}PR8koGAiaKUe z-mu0gSPv+`y{Q0Xh0&)!9f=lsijU^U(@XSI`+sz&Ct~(D3%SEff`9JuT+3NTQ+mWW zU=39gn5st*-K`wEXWrXD?D*bhl{IPBc;Cl>%~3g+i)xkbYmU&f#hr)Kx#oDA;?Aur zp3B48Iz654J=RgVIMIVgzeBrr=dU;Lu^`e+58;u7uU}~sL~Je%)N&Z&?hZ&_CcK}&?bL!8gxZHDY>}T(sg2s3@-n%Y8r4IM>(!~Owfgl8@kDFO zzB-&F)rGudr)~lx{w1liSe@N$z>*u`L#>&5l~?=s#>#ltD@G1wL}e5fBW3nrIhWyw z+A}I=2z)kLiz~Ek!)(4dbtN8iE`fvaZgZI$tkL$n*k15>M^YIyKx?|1pqbupc;=sU zH_hX))V7Yf(YTE+^{CiV;4D)*y0O`ej=cJtemhX!(xdcZrU5_10w&-$@H6^dWKW&tieDW77Hg={C zdzXhoh`5gd5nnsa`}sYP3Rhce-pHXS@U^pNdvad-e>2Dkr0&5?vGj+im^y4?@Es+{ zU|ua-<7eJeJ_Kwa%qxn(3(CD1XUqwS6D09kXI(KaH83179GfDnv)hoxkm?5sw7Z?* zWx*70K%{SPHg^eRHh0&y4>I-tz}WUl(?YsU8K&Oo{jTnD>hYp4fk-5K?Kw3*-$Z*1x^4S(~-TqsfvU-_jj7@96 zk7KalrY(cs)@ks5LqB)jMk9PWICz_~qz^{3=9QX1>&pwdj_VvM;h*Yror_1vnEEJ< z*sKO4Y{F2Cm!aKDD(+WDIhfuce>6JDDCvw8WK055d^_{AVGhoGGTE0VH5CA7C%eN@ zvC9{Hn+(#&@v2mP50Hbp`rQ|6e+~rMc8C&4YObapB?x|_uH*LnF^RtQ@8N!auA~bPY;L= z`Dx39Du`|#wXYXQ->L5gc3M?&3?JuZ2GX9{-@#|j!{rujVbTsFL5KIwmd((W=`dmV zMPU6<;V!T%TQT-F_R(pvjV! z|0X3F0XrQ7#`e$vA&i@{yS>2w9kklgtnhlt@?;EC8R%cyAG&);NE5q*aAjGF{1et6 zxXcmGL!iTlE&2FD!EX17q>uAcnsr1862wfhi2BRSn%#gR35a;H?lH}t_IRH#JD(lT zPJba}@MsNK zoi7I*I8R%p4VPa%cuQ|37seUKeLO~)ZjX>O_txE{C`)7@YxG4UroY!z3HlFbcyddE z+P^^}ps_M0LC-~3ULG(eVV@d_WB2a5SeH=vW9f?=R4KT8I5iXRs` zz?q5)g*`zPeW8Lz9zVsg`+hSWuI&kDxGQPk(;~#Y!j;pzTunN1^95n#pz+dX9wvdW zrP9w+LWRZ$DgH>SoISB#HjcRXTO^};J}*Fg;i zUGOu7h@MnNcYY`c%xivc{g8I|VjXE(BaPqf#;Au2Uc;(fOh?}j#4=N>N!ZE>QL3F$ zJgZhW1tpI}8G7xqYjjnjgX6c!Crb%e9V9Rymm9AZ#-ilOFd!I`{{O{k^x=vdQB)IX zr(eq~>xL+Y+#;!Y_$Pr>ZgXo|b%rjtnN~KhvWTsY<8aa=J>6~Q5df(<*3@c@#`HME z)QyljBt?-qRS`8E=gRt6I7@sHB0*)M90=3Z_EBCW{-_~&7(#~>(_|xtCV>1HJ`9Il zAU!uHUh4>+MzAA-uu+|HoWUYiv62{tQxX=1F@(kZB-?})YH-s=TX%K?kNGU4&L$Oj z2X7kisc#arA9c3%!l%e>80MoUl4Cg0ov6I^1>Y0j7r+_#N?1{C5`q$3ANuVxi#V`+ zujKNre5>p}IKG=hmcVwee2*+sg6bY`3)O%etSyg;kc`nde(BZEGkO&Tgyj?;8L)t! zk63izqmN8GU({SHX9+zE%$+ak`8PH6cdS%xOGZS)V!3V(Xx4jO>h1*8MZw2& zXe~^9x+BZ!c7wQ2k@%^cHgh`G?-ytUR#`(-^qR89U`X4U+a!jpX~QpU@eVxPTvNke z2?x)XxV8SkLuuB<*|=|wt8reG+73Qi`R}_e%zs`+jGa41(V;Onmx>wF91(fyodoD0lG6giAhNrg)j>3{N^J6-aX45BHs?*DM5Nsb@FRnE zM(5?~1x0A*D)m8LjTj+0C^>N-WTO@HMSxO(cS@M3&UtLljAF^a#c6ysy9~32dwq){ z9AY*QO;3!>L6Xo(mDdru+qEBy`>ed*)9+!Ls-J~YMXsdCRa!iBHpJ3%Xerwf4yRHk z1lgQ!n>K|Y(f4GE3-kQ1&)0T)g*7oo6Lq^2*=b=FW}|+X763U}$P}gMR`n%# z&(>K_Ln=6C^Yzx0LJ{^{f}+(0v@`b4#hbY&qO^Ug0+jkB0cuv4es3p;?hfi95C|S; z(IVcNT3NsCMn`RD`0mAbDRnlk0j4Fh2()RLB<)j*Zj=y@I$pbELoJT_4U2IVrOrXW z|Lowya=rd zEXgMJ-Q}?igNbc0g3R7P4e~n$_iD;-A4}`Uj=G~pleL1 zVa-+v^X(x@fj?n${hw|X<1%-}_EgycKV>J~)e%D#ZZp42z-~>{e;PqmL*nnwwI8-M z<|PdL$AjtKD`QHf&UX2LwC(+H0IMYLy&$+JkF2M=%gGP~UUI)99uphA3q~-^ntF7w zbpI$8S!jUU4GxACF=3z&yJ7OqKAZ3OE^4Zu^dqoj{W@7VK5fgNT0_78oiHnA3vWn_ zZab&vvVucU0}~P{16s?Zn-#cN#H!0h^}1GP=|r0txSKIK&9)ZEF!9u%jEDJG!p!d0 zc-ElZwbS?d@9oEI73T>M*1BVOlDk24qa1Qdl?9S?=%}#FaG}^d20zL3YW&`nfpTLW z6aUMh$m%*rK8zb}kO3gsA6}npx-Z8v3;sLVvN52pGis&zwCZQ}Ig`f;lA%442~||n z3BgXr<*?!N_idHm%p5lepO{d?4lb9TI2?%b^Jj1Ib7H`cNe*l$M{L9lVdTX%gYssS z-2ZkvjBWRp?kukHJoQx>1bz_*rR!T=>rX!WJ640QmMvwc$naW-ZpwV&SHE8Q=Y-(% z9yi97Jua&8hVi44dV!f4yCYsfUrG~0(kAhP_+97S9n+peEQMwE>oTcB)MV*_6Xeb4 zk)j=bGxU~TXc#s`zDX9COa-kV+)f!qDydzQ=}*S+F+%EBJ>IKWZ~gysZXK$^c91-* z6vJTx#rm!N<8llx`lf&Wr9OYf_pu!6mwoAEK0roMm+7@q8aw_T*F?;9#G^1A@VN_q zS+b0k-y>UT8&yC0s;o<*0rFEqD2-Lp6afJ0V&QF2RBzS=w=n!3>hat(Iq&!N+5VI1 zK^=<()s(bZF=h+z15azu?J;6Jce=UK70!-N*V++Ki9!98sB=ohH_;P1w#tu>>wnQX z%_(vasHF`9bE;^ivm2*l`&G}?vcWlZM=PgHCwk9^uCJA|Ug|YR*6GXLeXjX!4`b>t z*IGl8A&DzKFezfUDgx_>Y{xCVeoIO8@C++%ewY#Vleog1c_UEZjl+3Ge`M&cPPf51 zy6Z-hF!p)v&->rdg`neQy&HU*m#9UOL$h_j4CfI!7`XCa7vN-!W*L~(h!ZS=+ z@eU?RH7y7p@te|R9v;2qA(pcDRNmY5J34|+}`IVND?%iGn{)KS$o?>|XON%pW7)Z!pd$9y1KZxW7ZqEOM z!g?!CSZ$`LuaFDJCC7yg-x7MGR9Rn+Xj`y8jFaX9FjD>=<2G3_Tm0xqDhqe8!qzsy>pi?+vy=Z-$Df{)!52!dS<8^a1;ky?*5iWq(WhUWX zMEIykwjxAhuui2Y;>k$4gwTo^ewXUoub6QZWC!_<85u31d;!=yZMf}07o0Ip^&6|B;qe#g&_jiv4%DPrZ;J_t{V`3m2P&&CoL zKn2Nt#8?n0vw9vYd!*W@bexLme+cP9P%665T*bzA20xbhdP>rSoW*Hc%yO51_$i-&9+X4sAdQ*`5;{ zzjY%oZh3U=i;0!e@lVC&mAn_Zk!=PqgHDLZtkQoQC*(D^+pWRFb0K z{Imk4%(ch z@#Ik>F>wbMxBrp9jIp~R9TOZ$SjlA}*qBleI*Gw?DF~)b6<;_WAek=U?je#q>`L8m znRiVdT#?2cvCO@eZFFQTM&0vHIJxN=rbl@RBglg?3nv?Ov8H-<<0FIl)$IIo+aK`f z3MN=&#b6?m<+&iTU@L%>f`dp+XzuGcg`Q1?;-oI10!IS z03Q9bNyX*Rt-i?TJZ^jLf{ul9%slGVQeD95>_6Z>-ap5m4EE5uS`ed_*IQ|2t&-Nk zQRSiAC*Iw~m$Y)Dn_5imRckX!Za**}vfLXbXB4Y8C=TUFNur0ZEq99y#58+wP!;8G zC4S&2;ER7ge1!F^o@HOvVYG%B!KCQ?wSo}PdnR&HSLNnyAE7pygf>H|(hMrFj&JbS zlcLopYf3?k%CEt9_+&uQ-#u;#Tn+}qD#hE{utx^t{Zh4u0$S|l>l+Zidn$8kgN*HT z0G7INrVy|5Vj_Ndgzme0%xZ1#dn_{C;-$I>!XF03kBW=O;c#tBh1Am}{k?~8nd6#t zkO-~_9n7Eih>lf6Nw7El0l{703p9_HUlsl#McivW>mHnoDyis92P_hB5%R;{q;@f; z_Uq_+kBY?&-Hv^_M-P|E(WLZEt~ieivbc)g+lBKS%CB>Sa2Q|>+wC<09QIq#A@j&; zf@Rv;mvz$)OZ^E-D!g(MEUsV+WFG(+OhKc++XhNE4n7}?Kp3Ot&CYjXd{yEB%5ZE9 zkW7OizxE17&NF%VEIwTmej;L5`#lK|@*@WHCRI)8w~s3VILxiGCM7r_dg%JK8!3BF z@5-MX5CINwltgH2<-eqrBnvX5Q`c@E{4ATI(rogA zz2vnEhCRX-jle3i)66;Tweu?q1~W!3?BUz}YRNOEuvnoMz}ZSa+W-jcG6$Ya@F@nj zfaPN~R|+!NM{R*{4e8L6>Th+D9(Y{_;vo>`kt7@vR*k+V_k zk?ydGbQ_uq#NaBscy+bwK0W|2CW@k#=3t<$pZTHY>~yRP?6eTdC29C(Q<3^9v!L~|2))LL6axtgfUy#Ik2BkmzqN;{-UVdbs7S;2o{UvYR zZrYC6(oXQOG^Aai`{z-}kVnJT>o;}T#DPKG? zDJ-xhWnAyPv3M}YPOte!K%padR%#2Lr0@?}atZxAc5ccmBLIwG!-6g_)){v1`-0IrMSdtC&6lu0hXQf{(yhdLyaV3U&j)}Ie) z*+i`~@x6;0&{3BN{D9U5wZ$!kNiS^+8V%NDvTzu3BTD9_I`^QhVR_}G0E@@|A67|k zHz$D0(*+BUBkZ6TsK9Ttj+d1?N$L&IEy~ygb_Dd3Q!HZ8*!O+S3d}9JN;;;DxRiSd zl@T9=8=!4^3b4DG9dS}_fAj*Ng%iJYvIYfHa8EZp5^vDhl^vQGg4RpiVc1Up`S%nZ zGC1N2N3oW`-=3(w8w=3)S9ga1hqR+_BGZrW2TRv4$ubpoR04B*co%5oF(O4+3XT9t zrlQFVqU^6KkW7A>^jMbd(o7M|p-5=50`afSzIc(s`R6=tei*_jA$>O6LF(2Vd-Q?XC?M;=DFhcCvhcbP`0#H7d0zvS=fV zH!E2Y1NQz3&N1N0B`g{FzJiSI*Cn*L&t4;^RTxLWct=g@TCOLD+5b3fp(!-KFhaA6 zj%l@q@mK+e_w(E_sdW#EA|vpDHM@R3j2PvZL`MR@IpT{;*%(XL>6Mi&LiC|$y-gl@ zY1#rvU{1hgP|4^4eLxIkJN9nF$!>~7?q(t9kn?lAN2r<5bCe`mn*6mjp-2v7x-$o& zcXjhXQZBtE{0NJp3vbZ1$dRWe#d$)~dmufaZ} z0@@#hCPCUfGdQ|X8q_XgIZ@lwA){hKsAB^bv=X_1MorQ*}75RlMOf$@hv}~fd|4~+O`*D1CAt5kaEx!1{mSPT2JCBGAi?f_yJHbxM(XZ=2EMr?<*VUs(`R9{G|w;Isvp|0(gw1i5AP>O zk_jy3q>8dTvIRtcX1aP`_AMz?zHYY+n4ug_f3@w;EdqH2_`Uy}X!4^wO-0afGvxS! z`yhUF*k#~Bw*?^4J%@#9O*!p3vFH3aRj zXkRw(E}9Dgag+iei3&{rm(bnAyF2CIgCh2Hkv^gUHX6)J717(k>{`VBwt$c78iZ=U z8;#~XPQ-=F<&i1w|BRl+50Z+|8T)g5s;Akz>{_8n@}*`9RHgW!0I&D1vD-OkCJ%4H zbJ_lEC7Pb_$eWQxjc2H#F4$k20PUQbS$B^r)X+-db{khMpDRAqndc|6X&bHXo=*o- zq7rdym`?P`-Hs2ztVrA!xI%>$WfVh-@RRmI7(5Wawyt8n4^rbdpES)(*iPFesV%;O zEgo1z12V2c zjRiGQ6Rs{_YF)s=uamz^;fH8mYwd9ueO)FU=Q>F;uJ5Y*H;2n?1vxv6Pg@QD-NPwy zlg2wMl)u#32zHjZ6<(3u`dZ%fyKjaSCc|;WvK7E_|0k`f++#^TUPA;;I!rS(C>^s^ z4q$`sqYi*I6ETNp-KBV~FlQw?XyjjkDgw=r9Q3dO>L9L&+5Cr+5~&yw8kCO+bZU+F z-BEx>_%ngiFf$vUtJWa90(%hlv3j;%vI5{Z5sWHe83SNF+Kai|)X^HfsN92bVYuU5 z4Gg-1f52hYhmvsGfNH0&JVMVa1a6H6CHiLX(aXCB3Laj)f=M5%av)Q=y=J!fQ>fcvUSwh*V z)+HTu*9?Qf%15}vIZa5_KOoGE%PuvGWqO(G0}}p!2L3ci;DsAdDRjc?KI~ZEut!rJ zN|%DIFamO5rv!Y~r;=iN_vnvn1O!lgPUB? ziIPO%2;dmA=`y9Eb`QNa7%=6qUEvctQh;G*)+%nP2c$;S9COM%(#nFntD>ZZoatjE zMvpzz(*4j|bZ22!>{xBcebJ9JA?hlOY|vW^k&>jgt=K*b#z#g~iBU+% zKd^8(V!3{yL!JFumlRllo%PdGLI7eM2RrUSnRTln5HR&4)TO71l9rif9EDQX-%hV^ zB6s0&6O@GbX~YL|6!Jd~2LFgqwu;gq zHEXh3F$b5^H9gUR<#l*;iNuW})}k2Sk$S;NGVN73t!GI5!bPZ(2EA{49Ey~SW+8O# zj@|wv5?LZ~aSS|(B(xJ?;q=&6>y30>yZLZ%b;WI|0@C3t=Pw#CRnX%~Wr5+;OKShOi7CX(Pnp&=DCZLdcoelT>H>j+g-Z?a_tUMvh$&MNHlk)3IOC)a1lQSYtdBb)3L^A;oD zO1jEaqq%yK+VGW2xC@$idZkrh){zgeylI*Cx=S%6|0?|YlqnZ)MFilS2UF;*_N&_3 zX|Pi}S`~*9$-)i4ZTk)8>&eb$^8J4=Wp>rAe}gbmPtuvkE!_O~HHi2V$!1D=Y?5kh z35Fk<4^o5d$haGMV(fO%_*Ma6<>ssb%7| zRIWl>e1YHIgU`_O$L~Pp@n99zhAxtZApV;Vz}K>#mZJn%n}nr051g3)<^O>!zvHV| z*``)PTY z=C6@5A=DqKN_N_KtADX!o7xZb9q=`lJH^?;mS%%IfSLwA8n0YmUCb@WR~k!ys@&{w`Iuxrb*5D44g(I@8kQ=(e#!SP`=5e`26}_Hz@DQ${DOp;UwaFkvJh>o zBbzPcXnu({9AKs5_RQ+_pHBSz_eHd5n{LJc7qN zRbqHu=rI+#@yhq$1@On@3HGjE-#rN2B7%nJ>~=8&;`-Qi;rsCfZM-yrdZf@WN7{Aj zEn-OkC-W{3D5LHd9mguHfdKx5t-#88E`DR9FCtPtmg6wx#L@D@FOxfRnV5Ny#6}lH z(|fX}7W|v`DX`|+yj92zu=(|smt+Ubn)pa@)vAG9uqMWro%EA$P289LvtJ?)&V$sx zAjc_K6$6=T`@p~ zMOAwR-yjJiupgLL)z`Bbcl;3JI0P9hIA}Lgprgq&$KkW0E)MDY06HFHFK_zg7?`>5 zUaQI*1z_FKE6~GwwV}=0b4S9I+{C*8K5-CE8xltN^a`QRdV??Dv#bM zVCX+4KjS03vJ0%4&MqTLJH-im+A_6*Br?4J4Z}ESTvFy}=f(oS0mBH*7OeOx|6gg` zC#JexpxpIB#bYo9$3BLc>|Z{msb4d3fXk9WcQ`1)skN4hl9iZg^m->9kc+VU zTT?KdEp!rT$bLxDvDQ*3V)%)Y(S@H2s+V;XWQQTFTVz>gT5*gn#$6t0Q%F7l$Yw|g zqIMpZfBOCEOsd%dm%WlHa`LLK8gYhU@b7wvwOPhruBy6DwUF@R*X6;CBwHlMve|_# zVNRCmBh-SZA_#Rr!(m)9L?Vr5aS_jr1sS&4do^P+q;r0lZFMuU#ma!qT&L%m%>!1KQy0@gL>uRrA1zBX=@=+FDDO9&7v=9~&=G?B z_9pTso_Kp0s{14|$x&Q*DS#pJ*3xkvT1iqbaa*G0gwc+rCQt!zJedqIk{H)A`0#aR z^kwyIl7s1sWAlaCY#jyqFvEcp-F)BY;@z>~|I;6f%&H-dO)R|f%1g9r-PQ&GRYl{9 zXo$dTBY%gum}3LnV#aAvSvQHG>Fdd4?DoFU=VYyIJpi;4(iM8UyPyRP_#8Cn!6AJR z2`!$I=AJy1O1^%pbJ1osXkR<&)$-z}MmYUnvb)v@4&H7hJ(ai}B1v0@vr@tC6P zrSeSgsV|?6a_6Y@yxU&BV~RSQ1f}}2Jz)2Rx~#7x*r{WwhnP7Jj8TH2?N5n3#uJ0j zAXx3vTzc{Q?Rg7(w?Gmp{8ayQVG2(qZQ~|Lpr72JDS+D2D!IGQDx#{-OR1=lF-fw$ zlkfPcQ{*EB_s@8rj#BQA`6Im60u=v4Y(o1Yemo>cZEY;HXL=HBMF=(sCpNNJ#5dr@ znh#Y=J(R1xOVPGN(B*&DN>L%!r&lD(na5|nFm>E3!VnxT9>BF`#*^KZI?Ve0COdf! Xvc;lg`kR@Fan-e*Y!hB-sW^T=B453( literal 0 HcmV?d00001 diff --git a/src/main/scripts/ci/before-install.sh b/src/main/scripts/ci/before-install.sh new file mode 100644 index 00000000..75e8495c --- /dev/null +++ b/src/main/scripts/ci/before-install.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# get latest version of codacy reporter from sonatype +latest=$(curl "https://oss.sonatype.org/service/local/repositories/releases/content/com/codacy/codacy-coverage-reporter/maven-metadata.xml" | xpath -e "/metadata/versioning/release/text()") + +echo Downloading latest version $latest of codacy reporter from sonatype +# download laterst assembly jar +mvn dependency:get dependency:copy \ + -DoutputDirectory=$HOME \ + -DoutputAbsoluteArtifactFilename=true \ + -Dmdep.stripVersion=true \ + -DrepoUrl=https://oss.sonatype.org/service/local/repositories/releases/content/ \ + -Dartifact=com.codacy:codacy-coverage-reporter:$latest:jar:assembly + +echo local file md5sum: +md5sum ~/codacy-coverage-reporter-assembly.jar +echo remote file md5sum: +curl "https://oss.sonatype.org/service/local/repositories/releases/content/com/codacy/codacy-coverage-reporter/$latest/codacy-coverage-reporter-$latest-assembly.jar.md5" + diff --git a/style.importorder b/style.importorder new file mode 100644 index 00000000..883606f1 --- /dev/null +++ b/style.importorder @@ -0,0 +1,10 @@ +1=io.scalecube# +2=\# +3=io.scalecube +4=com +5=io +6=org +7=rx +8=java +9=javax + diff --git a/testlib/pom.xml b/testlib/pom.xml new file mode 100644 index 00000000..0604c0bd --- /dev/null +++ b/testlib/pom.xml @@ -0,0 +1,45 @@ + + + 4.0.0 + + io.scalecube + scalecube-parent + 1.0.9-SNAPSHOT + + scalecube-testlib + ScaleCube/Testlib + + + 4.11 + 1.9.5 + 2.6.2 + + + + + junit + junit + ${junit.version} + + + org.mockito + mockito-core + ${mockito.version} + + + org.slf4j + slf4j-api + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + + + + diff --git a/testlib/src/main/java/io/scalecube/testlib/BaseTest.java b/testlib/src/main/java/io/scalecube/testlib/BaseTest.java new file mode 100644 index 00000000..5cd8d24d --- /dev/null +++ b/testlib/src/main/java/io/scalecube/testlib/BaseTest.java @@ -0,0 +1,32 @@ +package io.scalecube.testlib; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Base test class. + * + * @author Anton Kharenko + */ +public class BaseTest { + + protected static final Logger LOGGER = LoggerFactory.getLogger(BaseTest.class); + + @Rule + public final TestName testName = new TestName(); + + @Before + public final void baseSetUp() throws Exception { + LOGGER.info("***** Test started : " + getClass().getSimpleName() + "." + testName.getMethodName() + " *****"); + } + + @After + public final void baseTearDown() throws Exception { + LOGGER.info("***** Test finished : " + getClass().getSimpleName() + "." + testName.getMethodName() + " *****"); + } + +} diff --git a/testlib/src/main/resources/log4j2.xml b/testlib/src/main/resources/log4j2.xml new file mode 100644 index 00000000..2b47cbff --- /dev/null +++ b/testlib/src/main/resources/log4j2.xml @@ -0,0 +1,23 @@ + + + + + + + %level{length=1} %date{MMdd-HHmm:ss,SSS} %logger{1.} %message [%thread]%n + + + + + + + + + + + + + + + + diff --git a/transport/pom.xml b/transport/pom.xml new file mode 100644 index 00000000..1be9a622 --- /dev/null +++ b/transport/pom.xml @@ -0,0 +1,105 @@ + + + 4.0.0 + + io.scalecube + scalecube-parent + 1.0.9-SNAPSHOT + + scalecube-transport + ScaleCube/Transport + + + + ${project.groupId} + scalecube-testlib + ${project.version} + test + + + io.scalecube + scalecube-utils + ${project.version} + + + + io.netty + netty-buffer + + + io.netty + netty-codec + + + io.netty + netty-common + + + io.netty + netty-handler + + + io.netty + netty-transport + + + io.netty + netty-transport-native-epoll + linux-x86_64 + + + io.reactivex + rxjava + + + com.google.guava + guava + + + com.google.code.findbugs + jsr305 + + + org.slf4j + slf4j-api + + + io.protostuff + protostuff-api + + + io.protostuff + protostuff-runtime + + + io.protostuff + protostuff-core + + + io.protostuff + protostuff-collectionschema + + + com.google.protobuf + protobuf-java + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.0.2 + + + + test-jar + + + + + + + + diff --git a/transport/src/main/java/io/scalecube/transport/BootstrapFactory.java b/transport/src/main/java/io/scalecube/transport/BootstrapFactory.java new file mode 100644 index 00000000..de3d276b --- /dev/null +++ b/transport/src/main/java/io/scalecube/transport/BootstrapFactory.java @@ -0,0 +1,115 @@ +package io.scalecube.transport; + +import com.google.common.base.Throwables; + +import io.netty.bootstrap.Bootstrap; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.buffer.PooledByteBufAllocator; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.epoll.EpollEventLoopGroup; +import io.netty.channel.epoll.EpollServerSocketChannel; +import io.netty.channel.epoll.EpollSocketChannel; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.ServerSocketChannel; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.util.concurrent.DefaultThreadFactory; +import io.netty.util.internal.SystemPropertyUtil; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Locale; +import java.util.concurrent.ThreadFactory; + +final class BootstrapFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(BootstrapFactory.class); + + private static boolean envSupportEpoll; + + static { + String name = SystemPropertyUtil.get("os.name").toLowerCase(Locale.UK).trim(); + if (!name.contains("linux")) { + envSupportEpoll = false; + LOGGER.warn("Env doesn't support epoll transport"); + } else { + try { + Class.forName("io.netty.channel.epoll.Native"); + envSupportEpoll = true; + LOGGER.info("Use epoll transport"); + } catch (Throwable t) { + LOGGER + .warn("Tried to use epoll transport, but it's not supported by host OS (or no corresponding libs included) " + + "using NIO instead, cause: " + Throwables.getRootCause(t)); + envSupportEpoll = false; + } + } + } + + private final TransportConfig config; + private final EventLoopGroup bossGroup; + private final EventLoopGroup workerGroup; + + public BootstrapFactory(TransportConfig config) { + this.config = config; + this.bossGroup = createEventLoopGroup(config.getBossThreads(), new DefaultThreadFactory("sc-boss", true)); + this.workerGroup = createEventLoopGroup(config.getWorkerThreads(), new DefaultThreadFactory("sc-io", true)); + } + + public ServerBootstrap serverBootstrap() { + ServerBootstrap bootstrap = new ServerBootstrap(); + bootstrap.group(bossGroup, workerGroup) + .channel(serverChannelClass()) + .childOption(ChannelOption.TCP_NODELAY, true) + .childOption(ChannelOption.SO_KEEPALIVE, true) + .childOption(ChannelOption.SO_REUSEADDR, true) + .childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT); + return bootstrap; + } + + public Bootstrap clientBootstrap() { + Bootstrap bootstrap = new Bootstrap(); + bootstrap.group(workerGroup) + .channel(channelClass()) + .option(ChannelOption.TCP_NODELAY, true) + .option(ChannelOption.SO_KEEPALIVE, true) + .option(ChannelOption.SO_REUSEADDR, true) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, config.getConnectTimeout()) + .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT); + return bootstrap; + } + + /** + * @return {@link EpollEventLoopGroup} or {@link NioEventLoopGroup} object dep on {@link #isEpollSupported()} call. + */ + private EventLoopGroup createEventLoopGroup(int threadNum, ThreadFactory threadFactory) { + return isEpollSupported() + ? new EpollEventLoopGroup(threadNum, threadFactory) + : new NioEventLoopGroup(threadNum, threadFactory); + } + + private Class serverChannelClass() { + return isEpollSupported() ? EpollServerSocketChannel.class : NioServerSocketChannel.class; + } + + private Class channelClass() { + return isEpollSupported() ? EpollSocketChannel.class : NioSocketChannel.class; + } + + private boolean isEpollSupported() { + return envSupportEpoll && config.isEnableEpoll(); + } + + public EventLoopGroup getWorkerGroup() { + return workerGroup; + } + + public void shutdown() { + this.bossGroup.shutdownGracefully(); + this.workerGroup.shutdownGracefully(); + } + +} diff --git a/transport/src/main/java/io/scalecube/transport/ExceptionHandler.java b/transport/src/main/java/io/scalecube/transport/ExceptionHandler.java new file mode 100644 index 00000000..05bf8137 --- /dev/null +++ b/transport/src/main/java/io/scalecube/transport/ExceptionHandler.java @@ -0,0 +1,29 @@ +package io.scalecube.transport; + +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Catching and logging exceptions. + *

+ * NOTE: this handler must be the last handler in the pipeline. + */ +@ChannelHandler.Sharable +final class ExceptionHandler extends ChannelDuplexHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(ExceptionHandler.class); + + @Override + public final void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + if (cause instanceof NetworkEmulatorException) { + LOGGER.debug("Network emulator fire exception for channel {}: {}", ctx.channel(), cause.getMessage()); + } else { + LOGGER.warn("Exception caught for channel {}, {}", ctx.channel(), cause.getMessage(), cause); + } + } + +} diff --git a/transport/src/main/java/io/scalecube/transport/Message.java b/transport/src/main/java/io/scalecube/transport/Message.java new file mode 100644 index 00000000..1b3d8c10 --- /dev/null +++ b/transport/src/main/java/io/scalecube/transport/Message.java @@ -0,0 +1,271 @@ +package io.scalecube.transport; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * The Class Message introduces generic protocol used for point to point communication by transport. + */ +public final class Message { + + /** + * This header is supposed to be used by application in case if same data type can be reused for several messages so + * it will allow to qualify the specific message type. + */ + public static final String HEADER_QUALIFIER = "q"; + + /** + * This header is supposed to be used by application in order to correlate request and response messages. + */ + public static final String HEADER_CORRELATION_ID = "cid"; + + /** + * This is a system header which used by transport for serialization and deserialization purpose. It is not supposed + * to be used by application directly and it is subject to changes in future releases. + */ + public static final String HEADER_DATA_TYPE = "_type"; + + private Map headers = Collections.emptyMap(); + private Object data; + private Address sender; + + /** + * Instantiates empty message for deserialization purpose. + */ + Message() {} + + private Message(Builder builder) { + this.data = builder.data(); + this.headers = builder.headers(); + } + + /** + * Instantiates a new message with the given data and without headers. + * + * @param data the data to build a message from + * @return the built message + */ + public static Message fromData(Object data) { + return withData(data).build(); + } + + /** + * Instantiates a new message builder with the given data and without headers. + * + * @param data the initial data for the builder + * @return a builder with initial data + */ + public static Builder withData(Object data) { + return builder().data(data); + } + + + /** + * Instantiates a new message with the given headers and with empty data. + * + * @param headers an initial headers to build a message from + * @return the built message + */ + public static Message fromHeaders(Map headers) { + return withHeaders(headers).build(); + } + + /** + * Instantiates a new message builder with the given headers and with empty data. + * + * @param headers the initial headers for the builder + * @return a builder with initial headers + */ + public static Builder withHeaders(Map headers) { + return builder().headers(headers); + } + + /** + * Instantiates a new message with the given qualifier header and with empty data. + * + * @param qualifier the qualifier to build a message from + * @return the built message + */ + public static Message fromQualifier(String qualifier) { + return withQualifier(qualifier).build(); + } + + /** + * Instantiates a new message builder with the given qualifier header and with empty data. + * + * @param qualifier the initial qualifier for the builder + * @return a builder with initial qualifier + */ + public static Builder withQualifier(String qualifier) { + return builder().qualifier(qualifier); + } + + /** + * Instantiates new message with the same data and headers as at given message. + * + * @param message the message to be copied + * @return a new message, with the same data and headers + */ + public static Message from(Message message) { + return with(message).build(); + } + + /** + * Instantiates new message builder with the same data and headers as at given message. + * + * @param message the message to instantiate the new builder from + * @return a builder with initial data and headers from the message + */ + public static Builder with(Message message) { + return withData(message.data).headers(message.headers); + } + + /** + * Instantiates new empty message builder. + * + * @return new builder + */ + public static Builder builder() { + return Builder.getInstance(); + } + + /** + * Sets data for deserialization purpose. + * + * @param data data to set + */ + void setData(Object data) { + this.data = data; + } + + /** + * Sets headers for deserialization purpose. + * + * @param headers headers to set + */ + void setHeaders(Map headers) { + checkArgument(headers != null); + this.headers = Collections.unmodifiableMap(headers); + } + + /** + * Sets sender and used by transport send method. + * + * @param sender address from where message was sent + */ + void setSender(Address sender) { + this.sender = sender; + } + + /** + * Returns the message headers. + * + * @return message headers + */ + public Map headers() { + return headers; + } + + /** + * Returns header value by given header name. + * + * @param name header name + * @return the message header by given header name + */ + public String header(String name) { + return headers.get(name); + } + + /** + * Returns message qualifier. + * + * @return qualifier string + */ + public String qualifier() { + return header(HEADER_QUALIFIER); + } + + /** + * Returns message correlation id. + * + * @return correlation id + */ + public String correlationId() { + return header(HEADER_CORRELATION_ID); + } + + /** + * Return the message data, which can be byte array, string or any type. + * + * @param data type + * @return payload of the message or null if message is without any payload + */ + public T data() { + // noinspection unchecked + return (T) data; + } + + /** + * Returns {@link Address} of the sender of this message. + * + * @return address + */ + public Address sender() { + return sender; + } + + @Override + public String toString() { + return "Message {headers: " + headers + ", sender: " + sender + ", data: " + data + '}'; + } + + public static class Builder { + + private Map headers = new HashMap<>(); + private Object data; + + private Builder() {} + + static Builder getInstance() { + return new Builder(); + } + + private Object data() { + return this.data; + } + + public Builder data(Object data) { + this.data = data; + return this; + } + + private Map headers() { + return this.headers; + } + + public Builder headers(Map headers) { + this.headers.putAll(headers); + return this; + } + + public Builder header(String key, String value) { + headers.put(key, value); + return this; + } + + public Builder qualifier(String qualifier) { + return header(HEADER_QUALIFIER, qualifier); + } + + public Builder correlationId(String correlationId) { + return header(HEADER_CORRELATION_ID, correlationId); + } + + public Message build() { + return new Message(this); + } + } +} diff --git a/transport/src/main/java/io/scalecube/transport/MessageCodec.java b/transport/src/main/java/io/scalecube/transport/MessageCodec.java new file mode 100644 index 00000000..34ca01c8 --- /dev/null +++ b/transport/src/main/java/io/scalecube/transport/MessageCodec.java @@ -0,0 +1,66 @@ +package io.scalecube.transport; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufInputStream; +import io.netty.buffer.ByteBufOutputStream; +import io.netty.handler.codec.DecoderException; +import io.netty.handler.codec.EncoderException; +import io.protostuff.ProtostuffIOUtil; +import io.protostuff.Schema; +import io.protostuff.runtime.RuntimeSchema; + +/** + * Contains static methods for message serializing/deserializing logic. + * + * @author Anton Kharenko + */ +public final class MessageCodec { + + private static final RecyclableLinkedBuffer recyclableLinkedBuffer = new RecyclableLinkedBuffer(); + + static { + // Register message schema + if (!RuntimeSchema.isRegistered(Message.class)) { + RuntimeSchema.register(Message.class, new MessageSchema()); + } + } + + private MessageCodec() { + // Do not instantiate + } + + /** + * Deserializes message from given byte buffer. + * + * @param bb byte buffer + */ + public static Message deserialize(ByteBuf bb) { + Schema schema = RuntimeSchema.getSchema(Message.class); + Message message = schema.newMessage(); + try { + ProtostuffIOUtil.mergeFrom(new ByteBufInputStream(bb), message, schema); + } catch (Exception e) { + throw new DecoderException(e.getMessage(), e); + } + + return message; + } + + /** + * Serializes given message into byte buffer. + * + * @param message message to serialize + * @param bb byte buffer of where to write serialzied message + */ + public static void serialize(Message message, ByteBuf bb) { + Schema schema = RuntimeSchema.getSchema(Message.class); + try (RecyclableLinkedBuffer rlb = recyclableLinkedBuffer.get()) { + try { + ProtostuffIOUtil.writeTo(new ByteBufOutputStream(bb), message, schema, rlb.buffer()); + } catch (Exception e) { + throw new EncoderException(e.getMessage(), e); + } + } + } + +} diff --git a/transport/src/main/java/io/scalecube/transport/MessageDeserializerHandler.java b/transport/src/main/java/io/scalecube/transport/MessageDeserializerHandler.java new file mode 100644 index 00000000..5de6052b --- /dev/null +++ b/transport/src/main/java/io/scalecube/transport/MessageDeserializerHandler.java @@ -0,0 +1,18 @@ +package io.scalecube.transport; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.MessageToMessageDecoder; + +import java.util.List; + +@ChannelHandler.Sharable +public final class MessageDeserializerHandler extends MessageToMessageDecoder { + + @Override + protected void decode(ChannelHandlerContext ctx, ByteBuf msg, List out) throws Exception { + out.add(MessageCodec.deserialize(msg)); + } + +} diff --git a/transport/src/main/java/io/scalecube/transport/MessageHandler.java b/transport/src/main/java/io/scalecube/transport/MessageHandler.java new file mode 100644 index 00000000..717e4d3f --- /dev/null +++ b/transport/src/main/java/io/scalecube/transport/MessageHandler.java @@ -0,0 +1,38 @@ +package io.scalecube.transport; + +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import rx.subjects.Subject; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Channel handler for getting message traffic. Activated when connection established/accepted. + *

+ * NOTE: in the pipeline this handler must be set just right before {@link ExceptionHandler}. + */ +@ChannelHandler.Sharable +final class MessageHandler extends ChannelInboundHandlerAdapter { + + private static final Logger LOGGER = LoggerFactory.getLogger(MessageHandler.class); + + private final Subject incomingMessagesSubject; + + MessageHandler(Subject incomingMessagesSubject) { + this.incomingMessagesSubject = incomingMessagesSubject; + } + + /** + * Publish {@code msg} on the incoming messages observable. + */ + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) { + Message message = (Message) msg; + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Received: {}", message); + } + incomingMessagesSubject.onNext(message); + } +} diff --git a/transport/src/main/java/io/scalecube/transport/MessageSchema.java b/transport/src/main/java/io/scalecube/transport/MessageSchema.java new file mode 100644 index 00000000..fa0bf88a --- /dev/null +++ b/transport/src/main/java/io/scalecube/transport/MessageSchema.java @@ -0,0 +1,230 @@ +package io.scalecube.transport; + +import static io.protostuff.LinkedBuffer.MIN_BUFFER_SIZE; +import static io.scalecube.transport.RecyclableLinkedBuffer.DEFAULT_MAX_CAPACITY; + +import com.google.common.collect.ImmutableMap; + +import io.protostuff.Input; +import io.protostuff.Output; +import io.protostuff.ProtostuffIOUtil; +import io.protostuff.Schema; +import io.protostuff.runtime.RuntimeSchema; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Protostuff schema for {@link Message}. + * + * @author Anton Kharenko + */ +final class MessageSchema implements Schema { + private static final Logger LOGGER = LoggerFactory.getLogger(MessageSchema.class); + + private static final int HEADER_KEYS_FIELD_NUMBER = 1; + private static final int HEADER_VALUES_FIELD_NUMBER = 2; + private static final int DATA_FIELD_NUMBER = 3; + private static final int SENDER_HOST_FIELD_NUMBER = 4; + private static final int SENDER_PORT_FIELD_NUMBER = 5; + + private static final RecyclableLinkedBuffer recyclableLinkedBuffer = + new RecyclableLinkedBuffer(MIN_BUFFER_SIZE, DEFAULT_MAX_CAPACITY); + + private static final Map fieldMap = ImmutableMap.builder() + .put("headerKeys", HEADER_KEYS_FIELD_NUMBER) + .put("headerValues", HEADER_VALUES_FIELD_NUMBER) + .put("data", DATA_FIELD_NUMBER) + .put("senderHost", SENDER_HOST_FIELD_NUMBER) + .put("senderPort", SENDER_PORT_FIELD_NUMBER) + .build(); + + private final Map> classCache = new ConcurrentHashMap<>(); + + @Override + public String getFieldName(int number) { + switch (number) { + case HEADER_KEYS_FIELD_NUMBER: + return "headerKeys"; + case HEADER_VALUES_FIELD_NUMBER: + return "headerValues"; + case DATA_FIELD_NUMBER: + return "data"; + case SENDER_HOST_FIELD_NUMBER: + return "senderHost"; + case SENDER_PORT_FIELD_NUMBER: + return "senderPort"; + default: + return null; + } + } + + @Override + public int getFieldNumber(String name) { + return fieldMap.get(name); + } + + @Override + public boolean isInitialized(Message message) { + return message != null; + } + + @Override + public Message newMessage() { + return new Message(); + } + + @Override + public String messageName() { + return Message.class.getSimpleName(); + } + + @Override + public String messageFullName() { + return Message.class.getName(); + } + + @Override + public Class typeClass() { + return Message.class; + } + + @Override + public void mergeFrom(Input input, Message message) throws IOException { + // Read input data + boolean iterate = true; + List headerKeys = new ArrayList<>(); + List headerValues = new ArrayList<>(); + String senderHost = null; + int senderPort = 0; + byte[] dataBytes = null; + while (iterate) { + int number = input.readFieldNumber(this); + switch (number) { + case 0: + iterate = false; + break; + case HEADER_KEYS_FIELD_NUMBER: + headerKeys.add(input.readString()); + break; + case HEADER_VALUES_FIELD_NUMBER: + headerValues.add(input.readString()); + break; + case DATA_FIELD_NUMBER: + dataBytes = input.readByteArray(); + break; + case SENDER_HOST_FIELD_NUMBER: + senderHost = input.readString(); + break; + case SENDER_PORT_FIELD_NUMBER: + senderPort = input.readInt32(); + break; + default: + input.handleUnknownField(number, this); + break; + } + } + + // Deserialize headers + Map headers = new HashMap<>(headerKeys.size()); + if (!headerKeys.isEmpty()) { + ListIterator headerValuesIterator = headerValues.listIterator(); + for (String key : headerKeys) { + String value = headerValuesIterator.next(); + headers.put(key, value); + } + } + + // Deserialize data + Object data = null; + if (dataBytes != null) { + String dataType = headers.get(Message.HEADER_DATA_TYPE); + if (dataType == null) { + data = dataBytes; + } else { + Optional optionalDataClass = classCache.computeIfAbsent(dataType, this::classForName); + if (optionalDataClass.isPresent()) { + headers.remove(Message.HEADER_DATA_TYPE); + Class dataClass = optionalDataClass.get(); + Schema dataSchema = RuntimeSchema.getSchema(dataClass); + data = dataSchema.newMessage(); + try { + ProtostuffIOUtil.mergeFrom(dataBytes, data, dataSchema); + } catch (Throwable e) { + LOGGER.error("Failed to deserialize : {}", message); + throw e; + } + } else { + data = dataBytes; + } + } + } + + // Deserialize sender + Address sender = senderHost != null ? Address.create(senderHost, senderPort) : null; + + // Set message + message.setHeaders(headers); + message.setData(data); + message.setSender(sender); + } + + @Override + public void writeTo(Output output, Message message) throws IOException { + // Write headers + if (!message.headers().isEmpty()) { + for (Map.Entry headerEntry : message.headers().entrySet()) { + if (headerEntry.getKey() != null && headerEntry.getValue() != null) { + output.writeString(HEADER_KEYS_FIELD_NUMBER, headerEntry.getKey(), true); + output.writeString(HEADER_VALUES_FIELD_NUMBER, headerEntry.getValue(), true); + } + } + } + + // Write data + Object originalData = message.data(); + if (originalData != null) { + if (originalData instanceof byte[]) { + // Write data byte array as is + output.writeByteArray(DATA_FIELD_NUMBER, (byte[]) originalData, false); + } else { + // Write data class as an additional header + Class dataClass = originalData.getClass(); + output.writeString(HEADER_KEYS_FIELD_NUMBER, Message.HEADER_DATA_TYPE, true); + output.writeString(HEADER_VALUES_FIELD_NUMBER, dataClass.getName(), true); + + // Write data as serialized byte array + Schema dataSchema = RuntimeSchema.getSchema(dataClass); + try (RecyclableLinkedBuffer rlb = recyclableLinkedBuffer.get()) { + byte[] array = ProtostuffIOUtil.toByteArray(originalData, dataSchema, rlb.buffer()); + output.writeByteArray(DATA_FIELD_NUMBER, array, false); + } + } + } + + // Write sender + Address sender = message.sender(); + if (sender != null) { + output.writeString(SENDER_HOST_FIELD_NUMBER, sender.host(), false); + output.writeInt32(SENDER_PORT_FIELD_NUMBER, sender.port(), false); + } + } + + private Optional classForName(String className) { + try { + Class dataClass = Class.forName(className); + return Optional.of(dataClass); + } catch (ClassNotFoundException e) { + return Optional.empty(); + } + } +} diff --git a/transport/src/main/java/io/scalecube/transport/MessageSerializerHandler.java b/transport/src/main/java/io/scalecube/transport/MessageSerializerHandler.java new file mode 100644 index 00000000..23477ac8 --- /dev/null +++ b/transport/src/main/java/io/scalecube/transport/MessageSerializerHandler.java @@ -0,0 +1,16 @@ +package io.scalecube.transport; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.MessageToByteEncoder; + +@ChannelHandler.Sharable +public final class MessageSerializerHandler extends MessageToByteEncoder { + + @Override + protected void encode(ChannelHandlerContext ctx, Message msg, ByteBuf out) throws Exception { + MessageCodec.serialize(msg, out); + } + +} diff --git a/transport/src/main/java/io/scalecube/transport/NetworkEmulator.java b/transport/src/main/java/io/scalecube/transport/NetworkEmulator.java new file mode 100644 index 00000000..e35157d6 --- /dev/null +++ b/transport/src/main/java/io/scalecube/transport/NetworkEmulator.java @@ -0,0 +1,223 @@ +package io.scalecube.transport; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetSocketAddress; +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Network Emulator is allowing to control link quality between endpoints in order to allow testing of message loss, + * message delay, cluster partitions cluster recovery and other network related conditions. + * + * @author Anton Kharenko + */ +public final class NetworkEmulator { + + private static final Logger LOGGER = LoggerFactory.getLogger(NetworkEmulator.class); + + public static final NetworkLinkSettings DEAD_LINK_SETTINGS = new NetworkLinkSettings(100, 0); + public static final NetworkLinkSettings ALIVE_LINK_SETTINGS = new NetworkLinkSettings(0, 0); + + private volatile NetworkLinkSettings defaultLinkSettings = ALIVE_LINK_SETTINGS; + + private final Map customLinkSettings = new ConcurrentHashMap<>(); + + private final AtomicLong totalMessageSentCount = new AtomicLong(); + + private final AtomicLong totalMessageLostCount = new AtomicLong(); + + private final boolean enabled; + + private final Address address; + + /** + * Creates new instance of network emulator. Should be always created internally by Transport. + * + * @param address local address + * @param enabled either network emulator is enabled + */ + NetworkEmulator(Address address, boolean enabled) { + this.address = address; + this.enabled = enabled; + } + + /** + * Returns link settings applied to the given destination. + * + * @param destination address of target endpoint + * @return network settings + */ + public NetworkLinkSettings getLinkSettings(Address destination) { + return customLinkSettings.getOrDefault(destination, defaultLinkSettings); + } + + /** + * Returns link settings applied to the given destination. + * + * @param address socket address of target endpoint + * @return network settings + */ + public NetworkLinkSettings getLinkSettings(InetSocketAddress address) { + // Check hostname:port + Address address1 = Address.create(address.getHostName(), address.getPort()); + if (customLinkSettings.containsKey(address1)) { + return customLinkSettings.get(address1); + } + + // Check ip:port + Address address2 = Address.create(address.getAddress().getHostAddress(), address.getPort()); + if (customLinkSettings.containsKey(address2)) { + return customLinkSettings.get(address2); + } + + // Use default + return defaultLinkSettings; + } + + /** + * Sets given network emulator settings for specific link. If network emulator is disabled do nothing. + * + * @param destination address of target endpoint + * @param lossPercent loss in percents + * @param meanDelay mean delay + */ + public void setLinkSettings(Address destination, int lossPercent, int meanDelay) { + if (!enabled) { + LOGGER.warn("Can't set network settings (loss={}%, mean={}ms) from {} to {} since network emulator is disabled", + lossPercent, meanDelay, address, destination); + return; + } + NetworkLinkSettings settings = new NetworkLinkSettings(lossPercent, meanDelay); + customLinkSettings.put(destination, settings); + LOGGER.info("Set network settings (loss={}%, mean={}ms) from {} to {}", + lossPercent, meanDelay, address, destination); + } + + /** + * Sets default network emulator settings. If network emulator is disabled do nothing. + * + * @param lossPercent loss in percents + * @param meanDelay mean delay + */ + public void setDefaultLinkSettings(int lossPercent, int meanDelay) { + if (!enabled) { + LOGGER.warn("Can't set default network settings (loss={}%, mean={}ms) for {} since network emulator is disabled", + lossPercent, meanDelay, address); + return; + } + defaultLinkSettings = new NetworkLinkSettings(lossPercent, meanDelay); + LOGGER.info("Set default network settings (loss={}%, mean={}ms) for {}", lossPercent, meanDelay, address); + } + + /** + * Blocks messages to the given destinations. If network emulator is disabled do nothing. + * + * @param destinations collection of target endpoints where to apply + */ + public void block(Address... destinations) { + block(Arrays.asList(destinations)); + } + + /** + * Blocks messages to the given destinations. If network emulator is disabled do nothing. + * + * @param destinations collection of target endpoints where to apply + */ + public void block(Collection

destinations) { + if (!enabled) { + LOGGER.warn("Can't block network from {} to {} since network emulator is disabled"); + return; + } + for (Address destination : destinations) { + customLinkSettings.put(destination, DEAD_LINK_SETTINGS); + } + LOGGER.info("Blocked network from {} to {}", address, destinations); + } + + /** + * Unblocks messages to given destinations. If network emulator is disabled do nothing. + * + * @param destinations collection of target endpoints where to apply + */ + public void unblock(Address... destinations) { + unblock(Arrays.asList(destinations)); + } + + /** + * Unblocks messages to given destinations. If network emulator is disabled do nothing. + * + * @param destinations collection of target endpoints where to apply + */ + public void unblock(Collection
destinations) { + if (!enabled) { + LOGGER.warn("Can't unblock network from {} to {} since network emulator is disabled", address, destinations); + return; + } + for (Address destination : destinations) { + customLinkSettings.remove(destination); + } + LOGGER.info("Unblocked network from {} to {}", address, destinations); + } + + /** + * Unblock messages to all destinations. If network emulator is disabled do nothing. + */ + public void unblockAll() { + if (!enabled) { + LOGGER.warn("Can't unblock network from {} since network emulator is disabled", address); + return; + } + customLinkSettings.clear(); + LOGGER.info("Unblocked all network from {}", address); + } + + /** + * Returns total message sent count computed by network emulator. If network emulator is disabled returns zero. + * + * @return total message sent + */ + public long totalMessageSentCount() { + if (!enabled) { + LOGGER.warn("Can't compute total messages sent from {} since network emulator is disabled", address); + return 0; + } + return totalMessageSentCount.get(); + } + + /** + * Returns total message lost count computed by network emulator. If network emulator is disabled returns zero. + * + * @return total message lost + */ + public long totalMessageLostCount() { + if (!enabled) { + LOGGER.warn("Can't compute total messages lost from {} since network emulator is disabled", address); + return 0; + } + return totalMessageLostCount.get(); + } + + // For internal use + void incrementMessageSentCount() { + if (!enabled) { + LOGGER.warn("since network emulator is disabled"); + return; + } + totalMessageSentCount.incrementAndGet(); + } + + // For internal use + void incrementMessageLostCount() { + if (!enabled) { + LOGGER.warn("since network emulator is disabled"); + return; + } + totalMessageLostCount.incrementAndGet(); + } + +} diff --git a/transport/src/main/java/io/scalecube/transport/NetworkEmulatorException.java b/transport/src/main/java/io/scalecube/transport/NetworkEmulatorException.java new file mode 100644 index 00000000..9b426ee2 --- /dev/null +++ b/transport/src/main/java/io/scalecube/transport/NetworkEmulatorException.java @@ -0,0 +1,21 @@ +package io.scalecube.transport; + +/** + * Exception which is thrown by network emulator on message loss. + * + * @author Anton Kharenko + */ +public final class NetworkEmulatorException extends RuntimeException { + + public NetworkEmulatorException(String message) { + super(message); + } + + /** + * No need for stack trace since those exceptions are not really an exceptions, but checked error conditions. + */ + @Override + public synchronized Throwable fillInStackTrace() { + return this; + } +} diff --git a/transport/src/main/java/io/scalecube/transport/NetworkEmulatorHandler.java b/transport/src/main/java/io/scalecube/transport/NetworkEmulatorHandler.java new file mode 100644 index 00000000..d4fd3ad1 --- /dev/null +++ b/transport/src/main/java/io/scalecube/transport/NetworkEmulatorHandler.java @@ -0,0 +1,68 @@ +package io.scalecube.transport; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelOutboundHandlerAdapter; +import io.netty.channel.ChannelPromise; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetSocketAddress; +import java.util.concurrent.Callable; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; + +@ChannelHandler.Sharable +final class NetworkEmulatorHandler extends ChannelOutboundHandlerAdapter { + + private static final Logger LOGGER = LoggerFactory.getLogger(NetworkEmulatorHandler.class); + + private final NetworkEmulator networkEmulator; + + public NetworkEmulatorHandler(NetworkEmulator networkEmulator) { + this.networkEmulator = networkEmulator; + } + + @Override + public void write(final ChannelHandlerContext ctx, final Object msg, final ChannelPromise promise) throws Exception { + NetworkLinkSettings networkSettings = resolveNetworkSettings(ctx.channel()); + networkEmulator.incrementMessageSentCount(); + + // Emulate message loss + boolean isLost = networkSettings.evaluateLoss(); + if (isLost) { + networkEmulator.incrementMessageLostCount(); + if (promise != null) { + promise.setFailure(new NetworkEmulatorException("NETWORK_BREAK detected, not sent " + msg)); + } + return; + } + + // Emulate message delay + int delay = (int) networkSettings.evaluateDelay(); + if (delay > 0) { + try { + ctx.channel().eventLoop().schedule((Callable) () -> { + ctx.writeAndFlush(msg, promise); + return null; + }, delay, TimeUnit.MILLISECONDS); + } catch (RejectedExecutionException e) { + String warn = "Rejected " + msg + " on " + ctx.channel(); + LOGGER.warn(warn, e); + if (promise != null) { + promise.setFailure(new NetworkEmulatorException(warn)); + } + } + return; + } + + super.write(ctx, msg, promise); + } + + private NetworkLinkSettings resolveNetworkSettings(Channel channel) { + InetSocketAddress remoteSocketAddress = (InetSocketAddress) channel.remoteAddress(); + return networkEmulator.getLinkSettings(remoteSocketAddress); + } +} diff --git a/transport/src/main/java/io/scalecube/transport/NetworkLinkSettings.java b/transport/src/main/java/io/scalecube/transport/NetworkLinkSettings.java new file mode 100644 index 00000000..b9ebbd75 --- /dev/null +++ b/transport/src/main/java/io/scalecube/transport/NetworkLinkSettings.java @@ -0,0 +1,78 @@ +package io.scalecube.transport; + +import java.util.concurrent.ThreadLocalRandom; + +/** + * This class contains settings for the network link and computations to evaluate message loss and message delay. + * Following parameters is present: + *
    + *
  • Percent of losing messages
    • + *
  • Mean network delays in milliseconds. Delays are emulated using exponential distribution of probabilities
    • + *
+ * + * @author Anton Kharenko + */ +public final class NetworkLinkSettings { + + private final int lossPercent; + private final int meanDelay; + + /** + * Constructor for link settings. + * + * @param lossPercent loss in percent + * @param mean mean dealy + */ + public NetworkLinkSettings(int lossPercent, int mean) { + this.lossPercent = lossPercent; + this.meanDelay = mean; + } + + /** + * Returns probability of message loss in percents. + * + * @return loss in percents + */ + public int lossPercent() { + return lossPercent; + } + + /** + * Returns mean network delay for message in milliseconds. + * + * @return mean delay + */ + public int meanDelay() { + return meanDelay; + } + + /** + * Indicator function telling is loss enabled. + * + * @return boolean indicating would loss occur + */ + public boolean evaluateLoss() { + return lossPercent > 0 && (lossPercent >= 100 || ThreadLocalRandom.current().nextInt(100) < lossPercent); + } + + /** + * Evaluates network delay according to exponential distribution of probabilities. + * + * @return delay + */ + public long evaluateDelay() { + if (meanDelay > 0) { + // Network delays (network delays). Delays should be emulated using exponential distribution of probabilities. + // log(1-x)/(1/mean) + Double x0 = ThreadLocalRandom.current().nextDouble(); + Double y0 = -Math.log(1 - x0) * meanDelay; + return y0.longValue(); + } + return 0; + } + + @Override + public String toString() { + return "NetworkLinkSettings{lossPercent=" + lossPercent + ", meanDelay=" + meanDelay + '}'; + } +} diff --git a/transport/src/main/java/io/scalecube/transport/RecyclableLinkedBuffer.java b/transport/src/main/java/io/scalecube/transport/RecyclableLinkedBuffer.java new file mode 100644 index 00000000..204e3592 --- /dev/null +++ b/transport/src/main/java/io/scalecube/transport/RecyclableLinkedBuffer.java @@ -0,0 +1,68 @@ +package io.scalecube.transport; + +import static com.google.common.base.Preconditions.checkState; + +import io.netty.util.Recycler; +import io.protostuff.LinkedBuffer; + +/** + * Facility class for {@link io.protostuff.LinkedBuffer}. Based on idea of object pooling (done vian + * {@link io.netty.util.Recycler}). + *

+ * Typical usage: + * + * 

+ *     RecycleableLinkedBuffer rlb = new RecycleableLinkedBuffer(bufferSize, maxCapacity)
+ *     LinkedBuffer lb = rlb.get();
+ *     ...
+ *     rlb.release();
+ *
+ */ +final class RecyclableLinkedBuffer implements AutoCloseable { + public static final int DEFAULT_BUFFER_SIZE = 512; + public static final int DEFAULT_MAX_CAPACITY = 256; + + private LinkedBuffer buffer; + private Recycler.Handle handle; + private final Recycler recycler; + + public RecyclableLinkedBuffer() { + this(DEFAULT_BUFFER_SIZE, DEFAULT_MAX_CAPACITY); + } + + /** + * @param bufferSize {@link io.protostuff.LinkedBuffer}'s buffer size. + * @param maxCapacity {@link io.netty.util.Recycler}'s. + */ + public RecyclableLinkedBuffer(final int bufferSize, int maxCapacity) { + this.recycler = new Recycler(maxCapacity) { + @Override + protected RecyclableLinkedBuffer newObject(Handle handle) { + RecyclableLinkedBuffer wrapper = new RecyclableLinkedBuffer(); + wrapper.buffer = LinkedBuffer.allocate(bufferSize); + wrapper.handle = handle; + return wrapper; + } + }; + } + + public LinkedBuffer buffer() { + checkState(buffer != null, "Call LinkedBufferWrapper.get() first"); + return buffer; + } + + public RecyclableLinkedBuffer get() { + return recycler.get(); + } + + public void release() { + checkState(buffer != null, "Call LinkedBufferWrapper.get() first"); + buffer.clear(); + recycler.recycle(this, handle); + } + + @Override + public void close() { + release(); + } +} diff --git a/transport/src/main/java/io/scalecube/transport/Transport.java b/transport/src/main/java/io/scalecube/transport/Transport.java new file mode 100644 index 00000000..01807acc --- /dev/null +++ b/transport/src/main/java/io/scalecube/transport/Transport.java @@ -0,0 +1,146 @@ +package io.scalecube.transport; + +import com.google.common.base.Throwables; + +import rx.Observable; + +import java.util.concurrent.CompletableFuture; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; + +/** + * Transport is responsible for maintaining existing p2p connections to/from other transports. It allows to send + * messages to other transports and listen for incoming messages. + */ +public interface Transport { + + /** + * Init transport with the default configuration synchronously. Starts to accept connections on local address. + * + * @return transport + */ + static Transport bindAwait() { + return bindAwait(TransportConfig.defaultConfig()); + } + + /** + * Init transport with the default configuration and network emulator flag synchronously. Starts to accept connections + * on local address. + * + * @return transport + */ + static Transport bindAwait(boolean useNetworkEmulator) { + return bindAwait(TransportConfig.builder().useNetworkEmulator(useNetworkEmulator).build()); + } + + /** + * Init transport with the given configuration synchronously. Starts to accept connections on local address. + * + * @return transport + */ + static Transport bindAwait(TransportConfig config) { + try { + return bind(config).get(); + } catch (Exception e) { + throw Throwables.propagate(Throwables.getRootCause(e)); + } + } + + /** + * Init transport with the default configuration asynchronously. Starts to accept connections on local address. + * + * @return promise for bind operation + */ + static CompletableFuture bind() { + return bind(TransportConfig.defaultConfig()); + } + + /** + * Init transport with the given configuration asynchronously. Starts to accept connections on local address. + * + * @param config transport config + * @return promise for bind operation + */ + static CompletableFuture bind(TransportConfig config) { + return new TransportImpl(config).bind0(); + } + + /** + * Returns local {@link Address} on which current instance of transport listens for incoming messages. + * + * @return address + */ + @Nonnull + Address address(); + + /** + * Stop transport, disconnect all connections and release all resources which belong to this transport. After + * transport is stopped it can't be used again. Observable returned from method {@link #listen()} will immediately + * emit onComplete event for all subscribers. + */ + void stop(); + + /** + * Stop transport, disconnect all connections and release all resources which belong to this transport. After + * transport is stopped it can't be opened again. Observable returned from method {@link #listen()} will immediately + * emit onComplete event for all subscribers. Stop is async operation, if result of operation is not needed use + * {@link Transport#stop}, otherwise pass {@link CompletableFuture}. + * + * @param promise promise will be completed with result of closing (void or exception) + */ + void stop(@CheckForNull CompletableFuture promise); + + + /** + * @return true if transport was stopped; false otherwise. + */ + boolean isStopped(); + + /** + * Sends message to the given address. It will issue connect in case if no transport channel by given transport + * {@code address} exists already. Send is an async operation. + * + * @param address address where message will be sent + * @param message message to send + * @throws IllegalArgumentException if {@code message} or {@code address} is null + */ + void send(@CheckForNull Address address, @CheckForNull Message message); + + /** + * Sends message to the given address. It will issue connect in case if no transport channel by given {@code address} + * exists already. Send is an async operation, if result of operation is not needed use + * {@link Transport#send(Address, Message)}, otherwise pass {@link CompletableFuture}. + * + * @param message message to send + * @param promise promise will be completed with result of sending (void or exception) + * @throws IllegalArgumentException if {@code message} or {@code address} is null + */ + void send(@CheckForNull Address address, @CheckForNull Message message, + @CheckForNull CompletableFuture promise); + + /** + * Returns stream of received messages. For each observers subscribed to the returned observable: + *
    + *
  • {@code rx.Observer#onNext(Object)} will be invoked when some message arrived to current transport
    • + *
  • {@code rx.Observer#onCompleted()} will be invoked when there is no possibility that server will receive new + * message observable for already closed transport
    • + *
  • {@code rx.Observer#onError(Throwable)} will not be invoked
    • + *
+ * + * @return Observable which emit received messages or complete event when transport is closed + */ + @Nonnull + Observable listen(); + + /** + * Returns network emulator associated with this instance of transport. It always returns non null instance even if + * network emulator is disabled by transport config. In case when network emulator is disable all calls to network + * emulator instance will result in no operation. + * + * @return network emulator + */ + @Nonnull + NetworkEmulator networkEmulator(); + +} diff --git a/transport/src/main/java/io/scalecube/transport/TransportConfig.java b/transport/src/main/java/io/scalecube/transport/TransportConfig.java new file mode 100644 index 00000000..716548bf --- /dev/null +++ b/transport/src/main/java/io/scalecube/transport/TransportConfig.java @@ -0,0 +1,214 @@ +package io.scalecube.transport; + +import javax.annotation.concurrent.Immutable; + +/** + * Encapsulate transport settings. + * + * @author Anton Kharenko + */ +@Immutable +public final class TransportConfig { + + public static final String DEFAULT_LISTEN_ADDRESS = null; + public static final String DEFAULT_LISTEN_INTERFACE = null; // Default listen settings fallback to getLocalHost + public static final boolean DEFAULT_PREFER_IP6 = false; + public static final int DEFAULT_PORT = 4801; + public static final int DEFAULT_PORT_COUNT = 100; + public static final boolean DEFAULT_PORT_AUTO_INCREMENT = true; + public static final int DEFAULT_CONNECT_TIMEOUT = 3000; + public static final boolean DEFAULT_USE_NETWORK_EMULATOR = false; + public static final boolean DEFAULT_ENABLE_EPOLL = true; + public static final int DEFAULT_BOSS_THREADS = 2; + public static final int DEFAULT_WORKER_THREADS = 0; + + private final String listenAddress; + private final String listenInterface; + private final boolean preferIPv6; + private final int port; + private final int portCount; + private final boolean portAutoIncrement; + private final int connectTimeout; + private final boolean useNetworkEmulator; + private final boolean enableEpoll; + private final int bossThreads; + private final int workerThreads; + + private TransportConfig(Builder builder) { + this.listenAddress = builder.listenAddress; + this.listenInterface = builder.listenInterface; + this.preferIPv6 = builder.preferIPv6; + this.port = builder.port; + this.portCount = builder.portCount; + this.portAutoIncrement = builder.portAutoIncrement; + this.connectTimeout = builder.connectTimeout; + this.useNetworkEmulator = builder.useNetworkEmulator; + this.enableEpoll = builder.enableEpoll; + this.bossThreads = builder.bossThreads; + this.workerThreads = builder.workerThreads; + } + + public static TransportConfig defaultConfig() { + return builder().build(); + } + + public static Builder builder() { + return new Builder(); + } + + public String getListenAddress() { + return listenAddress; + } + + public String getListenInterface() { + return listenInterface; + } + + public boolean isPreferIPv6() { + return preferIPv6; + } + + public int getPort() { + return port; + } + + public int getPortCount() { + return portCount; + } + + public boolean isPortAutoIncrement() { + return portAutoIncrement; + } + + public int getConnectTimeout() { + return connectTimeout; + } + + public boolean isUseNetworkEmulator() { + return useNetworkEmulator; + } + + public boolean isEnableEpoll() { + return enableEpoll; + } + + public int getBossThreads() { + return bossThreads; + } + + public int getWorkerThreads() { + return workerThreads; + } + + @Override + public String toString() { + return "TransportConfig{listenAddress=" + listenAddress + + ", listenInterface=" + listenInterface + + ", preferIPv6=" + preferIPv6 + + ", port=" + port + + ", portCount=" + portCount + + ", portAutoIncrement=" + portAutoIncrement + + ", connectTimeout=" + connectTimeout + + ", useNetworkEmulator=" + useNetworkEmulator + + ", enableEpoll=" + enableEpoll + + ", bossThreads=" + bossThreads + + ", workerThreads=" + workerThreads + + '}'; + } + + public static final class Builder { + + private String listenAddress = DEFAULT_LISTEN_ADDRESS; + private String listenInterface = DEFAULT_LISTEN_INTERFACE; + private boolean preferIPv6 = DEFAULT_PREFER_IP6; + private int port = DEFAULT_PORT; + private int portCount = DEFAULT_PORT_COUNT; + private boolean portAutoIncrement = DEFAULT_PORT_AUTO_INCREMENT; + private boolean useNetworkEmulator = DEFAULT_USE_NETWORK_EMULATOR; + private int connectTimeout = DEFAULT_CONNECT_TIMEOUT; + private boolean enableEpoll = DEFAULT_ENABLE_EPOLL; + private int bossThreads = DEFAULT_BOSS_THREADS; + private int workerThreads = DEFAULT_WORKER_THREADS; + + private Builder() {} + + /** + * Fills config with values equal to provided config. + * + * @param config trasport config + */ + public Builder fillFrom(TransportConfig config) { + this.listenAddress = config.listenAddress; + this.listenInterface = config.listenInterface; + this.preferIPv6 = config.preferIPv6; + this.port = config.port; + this.portCount = config.portCount; + this.portAutoIncrement = config.portAutoIncrement; + this.connectTimeout = config.connectTimeout; + this.useNetworkEmulator = config.useNetworkEmulator; + this.enableEpoll = config.enableEpoll; + this.bossThreads = config.bossThreads; + this.workerThreads = config.workerThreads; + return this; + } + + public Builder listenAddress(String listenAddress) { + this.listenAddress = listenAddress; + return this; + } + + public Builder listenInterface(String listenInterface) { + this.listenInterface = listenInterface; + return this; + } + + public Builder preferIPv6(boolean preferIPv6) { + this.preferIPv6 = preferIPv6; + return this; + } + + public Builder port(int port) { + this.port = port; + return this; + } + + public Builder portCount(int portCount) { + this.portCount = portCount; + return this; + } + + public Builder portAutoIncrement(boolean portAutoIncrement) { + this.portAutoIncrement = portAutoIncrement; + return this; + } + + public Builder connectTimeout(int connectTimeout) { + this.connectTimeout = connectTimeout; + return this; + } + + public Builder useNetworkEmulator(boolean useNetworkEmulator) { + this.useNetworkEmulator = useNetworkEmulator; + return this; + } + + public Builder enableEpoll(boolean enableEpoll) { + this.enableEpoll = enableEpoll; + return this; + } + + public Builder bossThreads(int bossThreads) { + this.bossThreads = bossThreads; + return this; + } + + public Builder workerThreads(int workerThreads) { + this.workerThreads = workerThreads; + return this; + } + + public TransportConfig build() { + return new TransportConfig(this); + } + } +} diff --git a/transport/src/main/java/io/scalecube/transport/TransportImpl.java b/transport/src/main/java/io/scalecube/transport/TransportImpl.java new file mode 100644 index 00000000..1b8ff961 --- /dev/null +++ b/transport/src/main/java/io/scalecube/transport/TransportImpl.java @@ -0,0 +1,326 @@ +package io.scalecube.transport; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static io.scalecube.transport.Addressing.MAX_PORT_NUMBER; +import static io.scalecube.transport.Addressing.MIN_PORT_NUMBER; + +import io.netty.bootstrap.Bootstrap; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.buffer.ByteBuf; +import io.netty.channel.Channel; +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.ServerChannel; +import io.netty.handler.codec.MessageToByteEncoder; +import io.netty.handler.codec.MessageToMessageDecoder; +import io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder; +import io.netty.handler.codec.protobuf.ProtobufVarint32LengthFieldPrepender; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import rx.Observable; +import rx.schedulers.Schedulers; +import rx.subjects.PublishSubject; +import rx.subjects.Subject; + +import java.net.BindException; +import java.net.InetAddress; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; + +final class TransportImpl implements Transport { + + private static final Logger LOGGER = LoggerFactory.getLogger(TransportImpl.class); + private static final CompletableFuture COMPLETED_PROMISE = CompletableFuture.completedFuture(null); + + private final TransportConfig config; + + private final Subject incomingMessagesSubject = PublishSubject.create().toSerialized(); + + private final Map outgoingChannels = new ConcurrentHashMap<>(); + + // Pipeline + private final BootstrapFactory bootstrapFactory; + private final IncomingChannelInitializer incomingChannelInitializer = new IncomingChannelInitializer(); + private final ExceptionHandler exceptionHandler = new ExceptionHandler(); + private final MessageToByteEncoder serializerHandler; + private final MessageToMessageDecoder deserializerHandler; + private final MessageHandler messageHandler; + + // Network emulator + private NetworkEmulator networkEmulator; + private NetworkEmulatorHandler networkEmulatorHandler; + + private Address address; + private ServerChannel serverChannel; + + private volatile boolean stopped = false; + + public TransportImpl(TransportConfig config) { + checkArgument(config != null); + this.config = config; + this.serializerHandler = new MessageSerializerHandler(); + this.deserializerHandler = new MessageDeserializerHandler(); + this.messageHandler = new MessageHandler(incomingMessagesSubject); + this.bootstrapFactory = new BootstrapFactory(config); + } + + /** + * Starts to accept connections on local address. + */ + public CompletableFuture bind0() { + ServerBootstrap server = bootstrapFactory.serverBootstrap().childHandler(incomingChannelInitializer); + + // Resolve listen IP address + InetAddress listenAddress = + Addressing.getLocalIpAddress(config.getListenAddress(), config.getListenInterface(), config.isPreferIPv6()); + + // Listen port + int bindPort = config.getPort(); + + return bind0(server, listenAddress, bindPort, bindPort + config.getPortCount()); + } + + /** + * Helper bind method to start accepting connections on {@code listenAddress} and {@code bindPort}. + * + * @param bindPort bind port. + * @param finalBindPort maximum port to bind. + * @throws NoSuchElementException if {@code bindPort} greater than {@code finalBindPort}. + * @throws IllegalArgumentException if {@code bindPort} doesnt belong to the range [{@link Addressing#MIN_PORT_NUMBER} + * .. {@link Addressing#MAX_PORT_NUMBER}]. + */ + private CompletableFuture bind0(ServerBootstrap server, InetAddress listenAddress, int bindPort, + int finalBindPort) { + + incomingMessagesSubject.subscribeOn(Schedulers.from(bootstrapFactory.getWorkerGroup())); + + final CompletableFuture result = new CompletableFuture<>(); + + // Perform basic bind port validation + if (bindPort < MIN_PORT_NUMBER || bindPort > MAX_PORT_NUMBER) { + result.completeExceptionally( + new IllegalArgumentException("Invalid port number: " + bindPort)); + return result; + } + if (bindPort > finalBindPort) { + result.completeExceptionally( + new NoSuchElementException("Could not find an available port from: " + bindPort + " to: " + finalBindPort)); + return result; + } + + // Get address object and bind + address = Address.create(listenAddress.getHostAddress(), bindPort); + ChannelFuture bindFuture = server.bind(listenAddress, address.port()); + bindFuture.addListener((ChannelFutureListener) channelFuture -> { + if (channelFuture.isSuccess()) { + serverChannel = (ServerChannel) channelFuture.channel(); + networkEmulator = new NetworkEmulator(address, config.isUseNetworkEmulator()); + networkEmulatorHandler = config.isUseNetworkEmulator() ? new NetworkEmulatorHandler(networkEmulator) : null; + LOGGER.info("Bound to: {}", address); + result.complete(TransportImpl.this); + } else { + Throwable cause = channelFuture.cause(); + if (config.isPortAutoIncrement() && isAddressAlreadyInUseException(cause)) { + LOGGER.warn("Can't bind to address {}, try again on different port [cause={}]", address, cause.toString()); + bind0(server, listenAddress, bindPort + 1, finalBindPort).thenAccept(result::complete); + } else { + LOGGER.error("Failed to bind to: {}, cause: {}", address, cause); + result.completeExceptionally(cause); + } + } + }); + return result; + } + + private boolean isAddressAlreadyInUseException(Throwable exception) { + return exception instanceof BindException + || (exception.getMessage() != null && exception.getMessage().contains("Address already in use")); + } + + @Override + @Nonnull + public Address address() { + return address; + } + + @Override + public boolean isStopped() { + return stopped; + } + + @Nonnull + @Override + public NetworkEmulator networkEmulator() { + return networkEmulator; + } + + @Override + public final void stop() { + stop(COMPLETED_PROMISE); + } + + @Override + public final void stop(CompletableFuture promise) { + checkState(!stopped, "Transport is stopped"); + checkArgument(promise != null); + stopped = true; + // Complete incoming messages observable + try { + incomingMessagesSubject.onCompleted(); + } catch (Exception ignore) { + // ignore + } + + // close connected channels + for (Address address : outgoingChannels.keySet()) { + ChannelFuture channelFuture = outgoingChannels.get(address); + if (channelFuture == null) { + continue; + } + if (channelFuture.isSuccess()) { + channelFuture.channel().close(); + } else { + channelFuture.addListener(ChannelFutureListener.CLOSE); + } + } + outgoingChannels.clear(); + + // close server channel + if (serverChannel != null) { + composeFutures(serverChannel.close(), promise); + } + + // TODO [AK]: shutdown boss/worker threads and listen for their futures + bootstrapFactory.shutdown(); + } + + @Nonnull + @Override + public final Observable listen() { + checkState(!stopped, "Transport is stopped"); + return incomingMessagesSubject.onBackpressureBuffer().asObservable(); + } + + @Override + public void send(@CheckForNull Address address, @CheckForNull Message message) { + send(address, message, COMPLETED_PROMISE); + } + + @Override + public void send(@CheckForNull Address address, @CheckForNull Message message, + @CheckForNull CompletableFuture promise) { + checkState(!stopped, "Transport is stopped"); + checkArgument(address != null); + checkArgument(message != null); + checkArgument(promise != null); + message.setSender(this.address); + + final ChannelFuture channelFuture = outgoingChannels.computeIfAbsent(address, this::connect); + if (channelFuture.isSuccess()) { + send(channelFuture.channel(), message, promise); + } else { + channelFuture.addListener((ChannelFuture chFuture) -> { + if (chFuture.isSuccess()) { + send(channelFuture.channel(), message, promise); + } else { + promise.completeExceptionally(chFuture.cause()); + } + }); + } + } + + private void send(Channel channel, Message message, CompletableFuture promise) { + if (promise == COMPLETED_PROMISE) { + channel.writeAndFlush(message, channel.voidPromise()); + } else { + composeFutures(channel.writeAndFlush(message), promise); + } + } + + /** + * Converts netty {@link ChannelFuture} to the given {@link CompletableFuture}. + * + * @param channelFuture netty channel future + * @param promise guava future; can be null + */ + private void composeFutures(ChannelFuture channelFuture, @Nonnull final CompletableFuture promise) { + channelFuture.addListener((ChannelFuture future) -> { + if (channelFuture.isSuccess()) { + promise.complete(channelFuture.get()); + } else { + promise.completeExceptionally(channelFuture.cause()); + } + }); + } + + private ChannelFuture connect(Address address) { + OutgoingChannelInitializer channelInitializer = new OutgoingChannelInitializer(address); + Bootstrap client = bootstrapFactory.clientBootstrap().handler(channelInitializer); + ChannelFuture connectFuture = client.connect(address.host(), address.port()); + + // Register logger and cleanup listener + connectFuture.addListener((ChannelFutureListener) channelFuture -> { + if (channelFuture.isSuccess()) { + LOGGER.debug("Connected from {} to {}: {}", TransportImpl.this.address, address, channelFuture.channel()); + } else { + LOGGER.warn("Failed to connect from {} to {}", TransportImpl.this.address, address); + outgoingChannels.remove(address); + } + }); + + return connectFuture; + } + + @ChannelHandler.Sharable + private final class IncomingChannelInitializer extends ChannelInitializer { + @Override + protected void initChannel(Channel channel) throws Exception { + ChannelPipeline pipeline = channel.pipeline(); + pipeline.addLast(new ProtobufVarint32FrameDecoder()); + pipeline.addLast(deserializerHandler); + pipeline.addLast(messageHandler); + pipeline.addLast(exceptionHandler); + } + } + + @ChannelHandler.Sharable + private final class OutgoingChannelInitializer extends ChannelInitializer { + private final Address address; + + public OutgoingChannelInitializer(Address address) { + this.address = address; + } + + @Override + protected void initChannel(Channel channel) throws Exception { + ChannelPipeline pipeline = channel.pipeline(); + pipeline.addLast(new ChannelDuplexHandler() { + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + LOGGER.debug("Disconnected from: {} {}", address, ctx.channel()); + outgoingChannels.remove(address); + super.channelInactive(ctx); + } + }); + pipeline.addLast(new ProtobufVarint32LengthFieldPrepender()); + pipeline.addLast(serializerHandler); + if (networkEmulatorHandler != null) { + pipeline.addLast(networkEmulatorHandler); + } + pipeline.addLast(exceptionHandler); + } + } +} diff --git a/transport/src/test/java/io/scalecube/transport/NetworkEmulatorTest.java b/transport/src/test/java/io/scalecube/transport/NetworkEmulatorTest.java new file mode 100644 index 00000000..79f1687c --- /dev/null +++ b/transport/src/test/java/io/scalecube/transport/NetworkEmulatorTest.java @@ -0,0 +1,44 @@ +package io.scalecube.transport; + +import org.junit.Assert; +import org.junit.Test; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; + +/** + * @author Anton Kharenko + */ +public class NetworkEmulatorTest { + + @Test + public void testResolveLinkSettingsBySocketAddress() throws UnknownHostException { + // Init network emulator + Address address = Address.from("localhost:1234"); + NetworkEmulator networkEmulator = new NetworkEmulator(address, true); + networkEmulator.setLinkSettings(Address.create("localhost", 5678), 25, 10); + networkEmulator.setLinkSettings(Address.create("192.168.0.1", 8765), 10, 20); + networkEmulator.setDefaultLinkSettings(0, 2); + + // Check resolve by hostname:port + InetSocketAddress addr1 = new InetSocketAddress("localhost", 5678); + NetworkLinkSettings link1 = networkEmulator.getLinkSettings(addr1); + Assert.assertEquals(25, link1.lossPercent()); + Assert.assertEquals(10, link1.meanDelay()); + + // Check resolve by ipaddr:port + byte[] byteAddr = new byte[]{(byte) 192, (byte) 168, 0, 1}; + InetSocketAddress addr2 = new InetSocketAddress(InetAddress.getByAddress("localhost", byteAddr), 8765); + NetworkLinkSettings link2 = networkEmulator.getLinkSettings(addr2); + Assert.assertEquals(10, link2.lossPercent()); + Assert.assertEquals(20, link2.meanDelay()); + + // Check default link settings + InetSocketAddress addr3 = new InetSocketAddress("localhost", 8765); + NetworkLinkSettings link3 = networkEmulator.getLinkSettings(addr3); + Assert.assertEquals(0, link3.lossPercent()); + Assert.assertEquals(2, link3.meanDelay()); + } + +} diff --git a/transport/src/test/java/io/scalecube/transport/TransportSendOrderTest.java b/transport/src/test/java/io/scalecube/transport/TransportSendOrderTest.java new file mode 100644 index 00000000..38ae705f --- /dev/null +++ b/transport/src/test/java/io/scalecube/transport/TransportSendOrderTest.java @@ -0,0 +1,238 @@ +package io.scalecube.transport; + +import static com.google.common.base.Throwables.propagate; +import static io.scalecube.transport.TransportTestUtils.createTransport; +import static io.scalecube.transport.TransportTestUtils.destroyTransport; +import static org.junit.Assert.assertEquals; + +import io.scalecube.testlib.BaseTest; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +import org.junit.After; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import rx.Subscriber; +import rx.observers.Subscribers; + +import java.util.ArrayList; +import java.util.List; +import java.util.LongSummaryStatistics; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.stream.LongStream; + +/** + * @author Anton Kharenko + */ +public class TransportSendOrderTest extends BaseTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(TransportSendOrderTest.class); + + // Auto-destroyed on tear down + private Transport client; + private Transport server; + + @After + public void tearDown() throws Exception { + destroyTransport(client); + destroyTransport(server); + } + + @Test + public void testSendOrderSingleThreadWithoutPromises() throws Exception { + server = createTransport(); + + int iterationNum = 11; // +1 warm up iteration + int sentPerIteration = 1000; + long[] iterationTimeSeries = new long[iterationNum - 1]; + for (int i = 0; i < iterationNum; i++) { + LOGGER.info("####### {} : iteration = {}", testName.getMethodName(), i); + + client = createTransport(); + final List received = new ArrayList<>(); + final CountDownLatch latch = new CountDownLatch(sentPerIteration); + + Subscriber serverSubscriber = Subscribers.create(message -> { + received.add(message); + latch.countDown(); + }); + server.listen().subscribe(serverSubscriber); + + long startAt = System.currentTimeMillis(); + for (int j = 0; j < sentPerIteration; j++) { + client.send(server.address(), Message.fromQualifier("q" + j)); + } + latch.await(20, TimeUnit.SECONDS); + long iterationTime = System.currentTimeMillis() - startAt; + if (i > 0) { // exclude warm up iteration + iterationTimeSeries[i - 1] = iterationTime; + } + assertSendOrder(sentPerIteration, received); + + LOGGER.info("Iteration time: {} ms", iterationTime); + + serverSubscriber.unsubscribe(); + destroyTransport(client); + } + + LongSummaryStatistics iterationTimeStats = LongStream.of(iterationTimeSeries).summaryStatistics(); + LOGGER.info("Iteration time stats (ms): {}", iterationTimeStats); + } + + @Test + public void testSendOrderSingleThread() throws Exception { + server = createTransport(); + + int iterationNum = 11; // +1 warm up iteration + int sentPerIteration = 1000; + long[] iterationTimeSeries = new long[iterationNum - 1]; + List totalSentTimeSeries = new ArrayList<>(sentPerIteration * (iterationNum - 1)); + for (int i = 0; i < iterationNum; i++) { + LOGGER.info("####### {} : iteration = {}", testName.getMethodName(), i); + List iterSentTimeSeries = new ArrayList<>(sentPerIteration); + + client = createTransport(); + final List received = new ArrayList<>(); + final CountDownLatch latch = new CountDownLatch(sentPerIteration); + + Subscriber serverSubscriber = Subscribers.create(message -> { + received.add(message); + latch.countDown(); + }); + server.listen().subscribe(serverSubscriber); + + long startAt = System.currentTimeMillis(); + for (int j = 0; j < sentPerIteration; j++) { + CompletableFuture sendPromise = new CompletableFuture<>(); + long sentAt = System.currentTimeMillis(); + client.send(server.address(), Message.fromQualifier("q" + j), sendPromise); + sendPromise.whenComplete((aVoid, exception) -> { + if (exception == null) { + long sentTime = System.currentTimeMillis() - sentAt; + iterSentTimeSeries.add(sentTime); + } else { + LOGGER.error("Failed to send message in {} ms", System.currentTimeMillis() - sentAt, exception); + } + }); + } + + latch.await(20, TimeUnit.SECONDS); + long iterationTime = System.currentTimeMillis() - startAt; + if (i > 0) { // exclude warm up iteration + iterationTimeSeries[i - 1] = iterationTime; + } + assertSendOrder(sentPerIteration, received); + + Thread.sleep(10); // await a bit for last msg confirmation + + LongSummaryStatistics iterSentTimeStats = iterSentTimeSeries.stream().mapToLong(v -> v).summaryStatistics(); + if (i == 0) { // warm up iteration + LOGGER.info("Warm up iteration time: {} ms", iterationTime); + LOGGER.info("Sent time stats warm up iter (ms): {}", iterSentTimeStats); + } else { + totalSentTimeSeries.addAll(iterSentTimeSeries); + LongSummaryStatistics totalSentTimeStats = totalSentTimeSeries.stream().mapToLong(v -> v).summaryStatistics(); + LOGGER.info("Iteration time: {} ms", iterationTime); + LOGGER.info("Sent time stats iter (ms): {}", iterSentTimeStats); + LOGGER.info("Sent time stats total (ms): {}", totalSentTimeStats); + } + + serverSubscriber.unsubscribe(); + destroyTransport(client); + } + + LongSummaryStatistics iterationTimeStats = LongStream.of(iterationTimeSeries).summaryStatistics(); + LOGGER.info("Iteration time stats (ms): {}", iterationTimeStats); + } + + @Test + public void testSendOrderMultiThread() throws Exception { + Transport server = createTransport(); + + final int total = 1000; + for (int i = 0; i < 10; i++) { + LOGGER.info("####### {} : iteration = {}", testName.getMethodName(), i); + ExecutorService exec = Executors.newFixedThreadPool(4, new ThreadFactoryBuilder().setDaemon(true).build()); + + Transport client = createTransport(); + final List received = new ArrayList<>(); + final CountDownLatch latch = new CountDownLatch(4 * total); + server.listen().subscribe(message -> { + received.add(message); + latch.countDown(); + }); + + Future f0 = exec.submit(sender(0, client, server.address(), total)); + Future f1 = exec.submit(sender(1, client, server.address(), total)); + Future f2 = exec.submit(sender(2, client, server.address(), total)); + Future f3 = exec.submit(sender(3, client, server.address(), total)); + + latch.await(20, TimeUnit.SECONDS); + + f0.get(1, TimeUnit.SECONDS); + f1.get(1, TimeUnit.SECONDS); + f2.get(1, TimeUnit.SECONDS); + f3.get(1, TimeUnit.SECONDS); + + exec.shutdownNow(); + + assertSenderOrder(0, total, received); + assertSenderOrder(1, total, received); + assertSenderOrder(2, total, received); + assertSenderOrder(3, total, received); + + destroyTransport(client); + } + + destroyTransport(client); + destroyTransport(server); + } + + private void assertSendOrder(int total, List received) { + ArrayList messages = new ArrayList<>(received); + assertEquals(total, messages.size()); + for (int k = 0; k < total; k++) { + assertEquals("q" + k, messages.get(k).qualifier()); + } + } + + private Callable sender(final int id, final Transport client, final Address address, final int total) { + return () -> { + for (int j = 0; j < total; j++) { + String correlationId = id + "/" + j; + CompletableFuture sendPromise = new CompletableFuture<>(); + client.send(address, Message.withQualifier("q").correlationId(correlationId).build(), sendPromise); + try { + sendPromise.get(3, TimeUnit.SECONDS); + } catch (Exception e) { + LOGGER.error("Failed to send message: j = {} id = {}", j, id, e); + propagate(e); + } + } + return null; + }; + } + + private void assertSenderOrder(int id, int total, List received) { + ArrayList messages = new ArrayList<>(received); + ArrayListMultimap group = ArrayListMultimap.create(); + for (Message message : messages) { + group.put(Integer.valueOf(message.correlationId().split("/")[0]), message); + } + + assertEquals(total, group.get(id).size()); + for (int k = 0; k < total; k++) { + assertEquals(id + "/" + k, group.get(id).get(k).correlationId()); + } + } + +} diff --git a/transport/src/test/java/io/scalecube/transport/TransportTest.java b/transport/src/test/java/io/scalecube/transport/TransportTest.java new file mode 100644 index 00000000..e6a72ce6 --- /dev/null +++ b/transport/src/test/java/io/scalecube/transport/TransportTest.java @@ -0,0 +1,450 @@ +package io.scalecube.transport; + +import static io.scalecube.transport.TransportTestUtils.createTransport; +import static io.scalecube.transport.TransportTestUtils.destroyTransport; +import static io.scalecube.transport.TransportTestUtils.send; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import io.scalecube.testlib.BaseTest; + +import org.junit.After; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import rx.Subscriber; + +import java.io.IOException; +import java.net.BindException; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public class TransportTest extends BaseTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(TransportTest.class); + + // Auto-destroyed on tear down + private Transport client; + private Transport server; + + @After + public void tearDown() throws Exception { + destroyTransport(client); + destroyTransport(server); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidListenConfig() { + Transport transport = null; + try { + TransportConfig config = TransportConfig.builder().listenInterface("eth0").listenAddress("10.10.10.10").build(); + transport = Transport.bindAwait(config); + } finally { + destroyTransport(transport); + } + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidListenInterface() { + Transport transport = null; + try { + TransportConfig config = TransportConfig.builder().listenInterface("yadayada").build(); + transport = Transport.bindAwait(config); + } finally { + destroyTransport(transport); + } + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidListenAddress() { + Transport transport = null; + try { + TransportConfig config = TransportConfig.builder().listenAddress("0.0.0.0").build(); + transport = Transport.bindAwait(config); + } finally { + destroyTransport(transport); + } + } + + @Test + public void testPortAutoIncrementRaceConditions() throws Exception { + int count = 30; + TransportConfig config = TransportConfig.builder() + .port(6000) + .portAutoIncrement(true) + .portCount(count) + .build(); + + Map, Boolean> transports = new ConcurrentHashMap<>(); + ExecutorService executor = Executors.newFixedThreadPool(4); + for (int i = 0; i < count; i++) { + executor.execute(() -> transports.put(Transport.bind(config), true)); + } + executor.shutdown(); + executor.awaitTermination(60, TimeUnit.SECONDS); + + CompletableFuture allFuturesResult = + CompletableFuture.allOf(transports.keySet().toArray(new CompletableFuture[transports.size()])); + + // Destroy transports + try { + allFuturesResult.get(60, TimeUnit.SECONDS); + } finally { + for (CompletableFuture transportFuture : transports.keySet()) { + if (transportFuture.isDone()) { + destroyTransport(transportFuture.get()); + } + } + } + } + + @Test + public void testBindExceptionWithoutPortAutoIncrement() throws Exception { + TransportConfig config = TransportConfig.builder() + .port(6000) + .portAutoIncrement(false) + .portCount(100) + .build(); + Transport transport1 = null; + Transport transport2 = null; + try { + transport1 = Transport.bindAwait(config); + transport2 = Transport.bindAwait(config); + fail("Didn't get expected bind exception"); + } catch (Throwable throwable) { + // Check that get address already in use exception + assertTrue(throwable instanceof BindException || throwable.getMessage().contains("Address already in use")); + } finally { + destroyTransport(transport1); + destroyTransport(transport2); + } + } + + @Test + public void testNoBindExceptionWithPortAutoIncrement() throws Exception { + TransportConfig config = TransportConfig.builder() + .port(6000) + .portAutoIncrement(true) + .portCount(100) + .build(); + Transport transport1 = null; + Transport transport2 = null; + + try { + transport1 = Transport.bindAwait(config); + transport2 = Transport.bindAwait(config); + } finally { + destroyTransport(transport1); + destroyTransport(transport2); + } + } + + @Test + public void testNoBindExceptionWithPortAutoIncrementWithHalfClosedSocket() throws Exception { + // Create half-closed socket scenario setup: server socket, connecting client socket, accepted socket + // on server side being closed, connected socket doesn't react on server's close + + ServerSocket serverSocket = new ServerSocket(6000); + Thread acceptor = new Thread(() -> { + while (true) { + Socket accepted = null; + try { + accepted = serverSocket.accept(); + accepted.close(); + } catch (Exception ignore) { + } + } + }); + acceptor.setDaemon(true); + acceptor.start(); + + Socket socket = new Socket(serverSocket.getInetAddress(), serverSocket.getLocalPort()); + + // Pretend that this port was chosen as bind port for the transport + int transportBindPort = socket.getLocalPort(); + + TransportConfig config = TransportConfig.builder() + .port(transportBindPort) + .portAutoIncrement(true) + .portCount(100) + .build(); + Transport transport1 = null; + try { + transport1 = Transport.bindAwait(config); + } finally { + destroyTransport(transport1); + serverSocket.close(); + socket.close(); + } + } + + @Test + public void testValidListenAddress() { + Transport transport = null; + try { + TransportConfig config = TransportConfig.builder().listenAddress("127.0.0.1").build(); + transport = Transport.bindAwait(config); + } finally { + destroyTransport(transport); + } + } + + @Test + public void testUnresolvedHostConnection() throws Exception { + client = createTransport(); + // create transport with wrong host + CompletableFuture sendPromise0 = new CompletableFuture<>(); + client.send(Address.from("wronghost:49255"), Message.fromData("q"), sendPromise0); + try { + sendPromise0.get(5, TimeUnit.SECONDS); + fail(); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + assertNotNull(cause); + assertEquals("Unexpected exception class", UnknownHostException.class, cause.getClass()); + } + } + + @Test + public void testInteractWithNoConnection() throws Exception { + Address serverAddress = Address.from("localhost:49255"); + for (int i = 0; i < 10; i++) { + LOGGER.info("####### {} : iteration = {}", testName.getMethodName(), i); + + client = createTransport(); + + // create transport and don't wait just send message + CompletableFuture sendPromise0 = new CompletableFuture<>(); + client.send(serverAddress, Message.fromData("q"), sendPromise0); + try { + sendPromise0.get(3, TimeUnit.SECONDS); + fail(); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + assertNotNull(cause); + assertTrue("Unexpected exception type (expects IOException)", cause instanceof IOException); + } + + // send second message: no connection yet and it's clear that there's no connection + CompletableFuture sendPromise1 = new CompletableFuture<>(); + client.send(serverAddress, Message.fromData("q"), sendPromise1); + try { + sendPromise1.get(3, TimeUnit.SECONDS); + fail(); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + assertNotNull(cause); + assertTrue("Unexpected exception type (expects IOException)", cause instanceof IOException); + } + + destroyTransport(client); + } + } + + @Test + public void testPingPongClientTFListenAndServerTFListen() throws Exception { + client = createTransport(); + server = createTransport(); + + server.listen().subscribe(message -> { + Address address = message.sender(); + assertEquals("Expected clientAddress", client.address(), address); + send(server, address, Message.fromQualifier("hi client")); + }); + + CompletableFuture messageFuture = new CompletableFuture<>(); + client.listen().subscribe(messageFuture::complete); + + send(client, server.address(), Message.fromQualifier("hello server")); + + Message result = messageFuture.get(3, TimeUnit.SECONDS); + assertNotNull("No response from serverAddress", result); + assertEquals("hi client", result.qualifier()); + } + + @Test + public void testNetworkSettings() throws InterruptedException { + client = createTransport(); + server = createTransport(); + + int lostPercent = 50; + int mean = 0; + client.networkEmulator().setLinkSettings(server.address(), lostPercent, mean); + + final List serverMessageList = new ArrayList<>(); + server.listen().subscribe(serverMessageList::add); + + int total = 1000; + for (int i = 0; i < total; i++) { + client.send(server.address(), Message.fromData("q" + i)); + } + + Thread.sleep(1000); + + int expectedMax = total / 100 * lostPercent + total / 100 * 5; // +5% for maximum possible lost messages + int size = serverMessageList.size(); + assertTrue("expectedMax=" + expectedMax + ", actual size=" + size, size < expectedMax); + } + + @Test + public void testPingPongOnSingleChannel() throws Exception { + server = createTransport(); + client = createTransport(); + + server.listen().buffer(2).subscribe(messages -> { + for (Message message : messages) { + Message echo = Message.fromData("echo/" + message.qualifier()); + server.send(message.sender(), echo); + } + }); + + final CompletableFuture> targetFuture = new CompletableFuture<>(); + client.listen().buffer(2).subscribe(targetFuture::complete); + + client.send(server.address(), Message.fromData("q1")); + client.send(server.address(), Message.fromData("q2")); + + List target = targetFuture.get(1, TimeUnit.SECONDS); + assertNotNull(target); + assertEquals(2, target.size()); + } + + @Test + public void testPingPongOnSeparateChannel() throws Exception { + server = createTransport(); + client = createTransport(); + + server.listen().buffer(2).subscribe(messages -> { + for (Message message : messages) { + Message echo = Message.fromData("echo/" + message.qualifier()); + server.send(message.sender(), echo); + } + }); + + final CompletableFuture> targetFuture = new CompletableFuture<>(); + client.listen().buffer(2).subscribe(targetFuture::complete); + + client.send(server.address(), Message.fromData("q1")); + client.send(server.address(), Message.fromData("q2")); + + List target = targetFuture.get(1, TimeUnit.SECONDS); + assertNotNull(target); + assertEquals(2, target.size()); + } + + @Test + public void testCompleteObserver() throws Exception { + server = createTransport(); + client = createTransport(); + + final CompletableFuture completeLatch = new CompletableFuture<>(); + final CompletableFuture messageLatch = new CompletableFuture<>(); + + server.listen().subscribe(new Subscriber() { + @Override + public void onCompleted() { + completeLatch.complete(true); + } + + @Override + public void onError(Throwable e) {} + + @Override + public void onNext(Message message) { + messageLatch.complete(message); + } + }); + + CompletableFuture send = new CompletableFuture<>(); + client.send(server.address(), Message.fromData("q"), send); + send.get(1, TimeUnit.SECONDS); + + assertNotNull(messageLatch.get(1, TimeUnit.SECONDS)); + + CompletableFuture close = new CompletableFuture<>(); + server.stop(close); + close.get(); + + assertTrue(completeLatch.get(1, TimeUnit.SECONDS)); + } + + @Test + public void testObserverThrowsException() throws Exception { + server = createTransport(); + client = createTransport(); + + server.listen().subscribe(message -> { + String qualifier = message.data(); + if (qualifier.startsWith("throw")) { + throw new RuntimeException("" + message); + } + if (qualifier.startsWith("q")) { + Message echo = Message.fromData("echo/" + message.qualifier()); + server.send(message.sender(), echo); + } + }, Throwable::printStackTrace); + + // send "throw" and raise exception on server subscriber + final CompletableFuture messageFuture0 = new CompletableFuture<>(); + client.listen().subscribe(messageFuture0::complete); + client.send(server.address(), Message.fromData("throw")); + Message message0 = null; + try { + message0 = messageFuture0.get(1, TimeUnit.SECONDS); + } catch (TimeoutException e) { + // ignore since expected behavior + } + assertNull(message0); + + // send normal message and check whether server subscriber is broken (no response) + final CompletableFuture messageFuture1 = new CompletableFuture<>(); + client.listen().subscribe(messageFuture1::complete); + client.send(server.address(), Message.fromData("q")); + Message transportMessage1 = null; + try { + transportMessage1 = messageFuture1.get(1, TimeUnit.SECONDS); + } catch (TimeoutException e) { + // ignore since expected behavior + } + assertNull(transportMessage1); + } + + @Test + public void testBlockAndUnblockTraffic() throws Exception { + client = createTransport(); + server = createTransport(); + + server.listen().subscribe(message -> server.send(message.sender(), message)); + + final List resp = new ArrayList<>(); + client.listen().subscribe(resp::add); + + // test at unblocked transport + send(client, server.address(), Message.fromQualifier("q/unblocked")); + + // then block client->server messages + Thread.sleep(1000); + client.networkEmulator().block(server.address()); + send(client, server.address(), Message.fromQualifier("q/blocked")); + + Thread.sleep(1000); + assertEquals(1, resp.size()); + assertEquals("q/unblocked", resp.get(0).qualifier()); + } + +} diff --git a/transport/src/test/java/io/scalecube/transport/TransportTestUtils.java b/transport/src/test/java/io/scalecube/transport/TransportTestUtils.java new file mode 100644 index 00000000..a7b3c869 --- /dev/null +++ b/transport/src/test/java/io/scalecube/transport/TransportTestUtils.java @@ -0,0 +1,57 @@ +package io.scalecube.transport; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +/** + * @author Anton Kharenko + */ +public final class TransportTestUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(TransportTestUtils.class); + + public static int CONNECT_TIMEOUT = 100; + public static int DEFAULT_PORT = 5800; + + private TransportTestUtils() { + // Do not instantiate + } + + public static Transport createTransport() { + TransportConfig config = TransportConfig.builder() + .connectTimeout(CONNECT_TIMEOUT) + .useNetworkEmulator(true) + .port(DEFAULT_PORT) + .build(); + return Transport.bindAwait(config); + } + + public static void destroyTransport(Transport transport) { + if (transport != null && !transport.isStopped()) { + CompletableFuture close = new CompletableFuture<>(); + transport.stop(close); + try { + close.get(1, TimeUnit.SECONDS); + } catch (Exception ignore) { + LOGGER.warn("Failed to await transport termination"); + } + } + } + + public static void send(final Transport from, final Address to, final Message msg) { + final CompletableFuture promise = new CompletableFuture<>(); + promise.thenAccept(aVoid -> { + if (promise.isDone()) { + try { + promise.get(); + } catch (Exception e) { + LOGGER.error("Failed to send {} to {} from transport: {}, cause: {}", msg, to, from, e.getCause()); + } + } + }); + from.send(to, msg, promise); + } +} diff --git a/travis-settings.xml b/travis-settings.xml new file mode 100644 index 00000000..cbdc4103 --- /dev/null +++ b/travis-settings.xml @@ -0,0 +1,26 @@ + + + + + ossrh + ${env.SONATYPE_USERNAME} + ${env.SONATYPE_PASSWORD} + + + + + ossrh + + true + + + gpg + ${env.GPG_KEY_NAME} + ${env.GPG_PASSPHRASE} + + + + diff --git a/utils/pom.xml b/utils/pom.xml new file mode 100644 index 00000000..be6bb7ab --- /dev/null +++ b/utils/pom.xml @@ -0,0 +1,30 @@ + + + 4.0.0 + + io.scalecube + scalecube-parent + 1.0.9-SNAPSHOT + + scalecube-utils + ScaleCube/Utils + + + + ${project.groupId} + scalecube-testlib + ${project.version} + test + + + + com.google.guava + guava + + + com.google.code.findbugs + jsr305 + + + + diff --git a/utils/src/main/java/io/scalecube/cluster/membership/IdGenerator.java b/utils/src/main/java/io/scalecube/cluster/membership/IdGenerator.java new file mode 100644 index 00000000..a1735065 --- /dev/null +++ b/utils/src/main/java/io/scalecube/cluster/membership/IdGenerator.java @@ -0,0 +1,72 @@ +package io.scalecube.cluster.membership; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; + +public final class IdGenerator { + + private static final int DEFAULT_SIZE = 10; + + public static final String DEFAULT_ALGORITHM = "MD5"; + + public static final ThreadLocal DIGEST_HOLDER = ThreadLocal.withInitial(IdGenerator::getDigest); + + private IdGenerator() { + // Do not instantiate + } + + public static String generateId() { + return generateId(DEFAULT_SIZE); + } + + public static String generateId(int length) { + return generateId(length, ThreadLocalRandom.current()); + } + + /** + * Generate and return a new session identifier. + * + * @param length The number of bytes to generate + * @return a new id string + */ + public static String generateId(int length, Random random) { + byte[] buffer = new byte[length]; + + int resultLenCounter = 0; + MessageDigest digest = DIGEST_HOLDER.get(); + int resultLen = length * 2; + char[] result = new char[resultLen]; + + while (resultLenCounter < resultLen) { + random.nextBytes(buffer); + buffer = digest.digest(buffer); + for (int j = 0; j < buffer.length && resultLenCounter < resultLen; j++) { + result[resultLenCounter++] = forHexDigit((buffer[j] & 0xf0) >> 4); + result[resultLenCounter++] = forHexDigit(buffer[j] & 0x0f); + } + } + + digest.reset(); + return new String(result); + } + + private static char forHexDigit(int digit) { + return (digit < 10) ? (char) ('0' + digit) : (char) ('A' - 10 + digit); + } + + /** + * Return the MessageDigest object to be used for calculating session identifiers. If none has been created yet, + * initialize one the first time this method is called. + * + * @return The hashing algorithm + */ + private static MessageDigest getDigest() { + try { + return MessageDigest.getInstance(DEFAULT_ALGORITHM); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("NoSuchAlgorithmException at getting instance for " + DEFAULT_ALGORITHM); + } + } +} diff --git a/utils/src/main/java/io/scalecube/transport/Address.java b/utils/src/main/java/io/scalecube/transport/Address.java new file mode 100644 index 00000000..355dc878 --- /dev/null +++ b/utils/src/main/java/io/scalecube/transport/Address.java @@ -0,0 +1,94 @@ +package io.scalecube.transport; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.base.Strings; + +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import javax.annotation.concurrent.Immutable; + +@Immutable +public final class Address { + + private static final Pattern ADDRESS_FORMAT = Pattern.compile("(?^.*):(?\\d+$)"); + + private final String host; + private final int port; + + private Address(@CheckForNull String host, int port) { + checkArgument(!Strings.isNullOrEmpty(host)); + this.host = host; + this.port = port; + } + + /** + * Parses given string to create address instance. For localhost variant host may come in: {@code 127.0.0.1}, + * {@code localhost}; when localhost case detected then node's public IP address would be resolved. + * + * @param hostAndPort must come in form {@code host:port} + */ + public static Address from(@CheckForNull String hostAndPort) { + checkArgument(!Strings.isNullOrEmpty(hostAndPort)); + + Matcher matcher = ADDRESS_FORMAT.matcher(hostAndPort); + if (!matcher.find()) { + throw new IllegalArgumentException(); + } + + String host = matcher.group(1); + checkArgument(!Strings.isNullOrEmpty(host)); + String host1 = + "localhost".equals(host) || "127.0.0.1".equals(host) ? Addressing.getLocalIpAddress().getHostAddress() : host; + int port = Integer.parseInt(matcher.group(2)); + return new Address(host1, port); + } + + /** + * Creates address from host and port. + */ + public static Address create(String host, int port) { + return new Address(host, port); + } + + /** + * Host address. + */ + @Nonnull + public String host() { + return host; + } + + /** + * Port. + */ + public int port() { + return port; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + Address that = (Address) other; + return Objects.equals(host, that.host) && Objects.equals(port, that.port); + } + + @Override + public int hashCode() { + return Objects.hash(host, port); + } + + @Override + public String toString() { + return host + ":" + port; + } +} diff --git a/utils/src/main/java/io/scalecube/transport/Addressing.java b/utils/src/main/java/io/scalecube/transport/Addressing.java new file mode 100644 index 00000000..bc392090 --- /dev/null +++ b/utils/src/main/java/io/scalecube/transport/Addressing.java @@ -0,0 +1,160 @@ +package io.scalecube.transport; + +import com.google.common.base.Strings; +import com.google.common.base.Throwables; + +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.net.UnknownHostException; +import java.util.Collections; +import java.util.Enumeration; +import java.util.List; + +/** + * Utility class which finds local IP address and currently available server ports. + * + * @author Anton Kharenko + * @author Artem Vysochyn + */ +public final class Addressing { + + /** + * The minimum server port number. Set at 1100 to avoid returning privileged port numbers. + */ + public static final int MIN_PORT_NUMBER = 1100; + + /** + * The maximum server port number. + */ + public static final int MAX_PORT_NUMBER = 65535; + + private Addressing() { + // Do not instantiate + } + + /** + * Returns {@link InetAddress} by given IP address or network interface name. + * + * @param listenAddress listen address; if set then {@code listenInterface} must be not set. + * @param listenInterface network interface name; if set then {@code listenAddress} must be not set. + * @param preferIPv6 should we prefer IPv6 when choosing IP address; accounted when {@code listenInterface} is set. + * @return {@link InetAddress} object. + * @throws IllegalArgumentException if both {@code listenAddress} and {@code listenInterface} were passed. + * @throws IllegalArgumentException if {@code listenAddress} can't be resolved or if it reprensents wildcard address, + * or if it doesn't belong to any active network interface. + */ + public static InetAddress getLocalIpAddress(String listenAddress, String listenInterface, boolean preferIPv6) { + InetAddress ipAddress; + if (!Strings.isNullOrEmpty(listenAddress) && !Strings.isNullOrEmpty(listenInterface)) { + throw new IllegalArgumentException("Not allowed to set both listenAddress and listenInterface, choose one"); + } else if (!Strings.isNullOrEmpty(listenAddress)) { + try { + ipAddress = InetAddress.getByName(listenAddress); + } catch (UnknownHostException e) { + throw new IllegalArgumentException("Unknown listenAddress: " + listenAddress); + } + // account that 0.0.0.0 is not allowed + if (ipAddress.isAnyLocalAddress()) { + throw new IllegalArgumentException("listenAddress: " + listenAddress + " cannot be a wildcard address"); + } + // ensure address is valid + if (!isValidLocalIpAddress(ipAddress)) { + throw new IllegalArgumentException( + "listenAddress: " + listenAddress + " doesn't belong to any active network interface"); + } + } else if (!Strings.isNullOrEmpty(listenInterface)) { + ipAddress = getNetworkInterfaceIpAddress(listenInterface, preferIPv6); + } else { + // fallback to local ip address + ipAddress = getLocalIpAddress(); + } + return ipAddress; + } + + /** + * Getting local IP address by the address of local host. NOTE: returned IP address is expected to be a + * publicly visible IP address. + * + * @throws RuntimeException wrapped {@link UnknownHostException} in case when local host name couldn't be resolved + * into an address. + */ + public static InetAddress getLocalIpAddress() { + try { + return InetAddress.getLocalHost(); + } catch (UnknownHostException e) { + throw Throwables.propagate(e); + } + } + + /** + * @return {@link InetAddress} by network interface name and a flag indicating whether returned IP address will be + * IPv4 or IPv6. + */ + private static InetAddress getNetworkInterfaceIpAddress(String listenInterface, boolean preferIPv6) { + try { + NetworkInterface ni = NetworkInterface.getByName(listenInterface); + if (ni == null) { + throw new IllegalArgumentException("Configured network interface: " + listenInterface + " could not be found"); + } + if (!ni.isUp()) { + throw new IllegalArgumentException("Configured network interface: " + listenInterface + " is not active"); + } + Enumeration addrs = ni.getInetAddresses(); + if (!addrs.hasMoreElements()) { + throw new IllegalArgumentException( + "Configured network interface: " + listenInterface + " was found, but had no addresses"); + } + // try to return the first address of the preferred type, otherwise return the first address + InetAddress result = null; + while (addrs.hasMoreElements()) { + InetAddress addr = addrs.nextElement(); + if (preferIPv6 && addr instanceof Inet6Address) { + return addr; + } + if (!preferIPv6 && addr instanceof Inet4Address) { + return addr; + } + if (result == null) { + result = addr; + } + } + return result; + } catch (SocketException e) { + throw new IllegalArgumentException( + "Configured network interface: " + listenInterface + " caused an exception", e); + } + } + + /** + * @return boolean indicating whether given address belongs to any active network interface. + */ + private static boolean isValidLocalIpAddress(InetAddress listenAddress) { + List networkInterfaces; + try { + networkInterfaces = Collections.list(NetworkInterface.getNetworkInterfaces()); + } catch (SocketException e) { + throw new IllegalArgumentException("Can't get list of network interfaces", e); + } + // go and check out network interface by IP address + for (NetworkInterface ni : networkInterfaces) { + try { + if (ni.isUp()) { + Enumeration addrs = ni.getInetAddresses(); + while (addrs.hasMoreElements()) { + InetAddress addr = addrs.nextElement(); + if (addr.getHostAddress().equals(listenAddress.getHostAddress())) { + return true; + } + } + } + } catch (SocketException e) { + throw new IllegalArgumentException("Network interface: " + ni + " caused an exception", e); + } + } + // looked at all network interfaces and didn't match IP address + return false; + } +} diff --git a/utils/src/test/java/io/scalecube/utils/AddressTest.java b/utils/src/test/java/io/scalecube/utils/AddressTest.java new file mode 100644 index 00000000..022d4fcb --- /dev/null +++ b/utils/src/test/java/io/scalecube/utils/AddressTest.java @@ -0,0 +1,33 @@ +package io.scalecube.utils; + +import static org.junit.Assert.assertEquals; + +import io.scalecube.testlib.BaseTest; +import io.scalecube.transport.Address; +import io.scalecube.transport.Addressing; + +import org.junit.Test; + +public class AddressTest extends BaseTest { + + @Test + public void testParseHostPort() throws Exception { + Address address1 = Address.from("localhost:5810"); + assertEquals(5810, address1.port()); + assertEquals(Addressing.getLocalIpAddress().getHostAddress(), address1.host()); + + Address address2 = Address.from("127.0.0.1:5810"); + assertEquals(5810, address1.port()); + assertEquals(Addressing.getLocalIpAddress().getHostAddress(), address2.host()); + + assertEquals(address1, address2); + } + + @Test + public void testParseUnknownHostPort() throws Exception { + Address address = Address.from("host:1111"); + assertEquals(1111, address.port()); + assertEquals("host", address.host()); + } + +} diff --git a/utils/src/test/java/io/scalecube/utils/IdGeneratorTest.java b/utils/src/test/java/io/scalecube/utils/IdGeneratorTest.java new file mode 100644 index 00000000..e4837ad0 --- /dev/null +++ b/utils/src/test/java/io/scalecube/utils/IdGeneratorTest.java @@ -0,0 +1,33 @@ +package io.scalecube.utils; + +import io.scalecube.cluster.membership.IdGenerator; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class IdGeneratorTest { + + private final static int ATTEMPTS = (int) 2e+6; + + @Test + public void generatorUniquenessTest() { + Map previds = new HashMap<>(ATTEMPTS); + + for (int attemptNumber = 0; attemptNumber < ATTEMPTS; attemptNumber++) { + String id = generateId(); + if (previds.containsKey(id)) { + Assert.fail("Found key duplication on attempt " + attemptNumber + + " same id = " + id + " as at attempt " + previds.get(id)); + } else { + previds.put(id, attemptNumber); + } + } + } + + private String generateId() { + return IdGenerator.generateId(10); + } +}