From 32683978e4e73a77f536ec61615172ae460c7603 Mon Sep 17 00:00:00 2001 From: Mahesh Kumar Gangula Date: Tue, 5 Jan 2021 21:55:19 +0530 Subject: [PATCH 1/6] Issue #000 fix: remove invalid jobs configuration. --- .../roles/flink-jobs-deploy/defaults/main.yml | 21 ---- .../helm_charts/datapipeline_jobs/values.j2 | 105 ------------------ 2 files changed, 126 deletions(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 36fb9c771e..4e867d1892 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -125,27 +125,6 @@ flink_job_names: taskmanager_memory: 1024m taskslots: 1 cpu_requests: 0.3 - certificate-generator: - job_class_name: 'org.sunbird.job.task.CertificateGeneratorStreamTask' - replica: 1 - jobmanager_memory: 1024m - taskmanager_memory: 1024m - taskslots: 1 - cpu_requests: 0.3 - post-certificate-processor: - job_class_name: 'org.sunbird.job.task.PostCertificateProcessorStreamTask' - replica: 1 - jobmanager_memory: 1024m - taskmanager_memory: 1024m - taskslots: 1 - cpu_requests: 0.3 - certificate-pre-processor: - job_class_name: 'org.sunbird.job.task.CertificatePreProcessorStreamTask' - replica: 1 - jobmanager_memory: 1024m - taskmanager_memory: 1024m - taskslots: 1 - cpu_requests: 0.3 ### Global vars middleware_course_keyspace: "sunbird_courses" diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 35ee19b090..2e32435f92 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -192,108 +192,3 @@ post-publish-processor: jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 -certificate-generator: - certificate-generator: |+ - include file("/data/flink/conf/base-config.conf") - kafka { - input.topic = {{ env_name }}.generate.certificate.request - output.failed.topic = {{ env_name }}.generate.certificate.failed - output.post.certificate.processor.topic = {{ env_name }}.post.certificate.process.request - groupId = {{ env_name }}-generator-certificate-group - } - task { - consumer.parallelism = {{ certificate_generator_consumer_parallelism }} - parallelism = {{ certificate_generator_parallelism }} - } - cert-reg { - basePath = "{{ cert_reg_service_base_url }}" - } - enc-service { - basePath = "{{ enc_service_base_url }}" - } - cert_domain_url = "{{ cert_domain_url }}" - cert_container_name = "{{ cert_container_name }}" - cert_cloud_storage_type = "{{ cert_cloud_storage_type }}" - cert_azure_storage_secret = "{{ cert_azure_storage_secret }}" - cert_azure_storage_key = "{{ cert_azure_storage_key }}" - - flink-conf: |+ - jobmanager.memory.flink.size: {{ flink_job_names['certificate-generator'].jobmanager_memory }} - taskmanager.memory.flink.size: {{ flink_job_names['certificate-generator'].taskmanager_memory }} - taskmanager.numberOfTaskSlots: {{ flink_job_names['certificate-generator'].taskslots }} - parallelism.default: 1 - jobmanager.execution.failover-strategy: region - taskmanager.memory.network.fraction: 0.1 - -post-certificate-processor: - post-certificate-processor: |+ - include file("/data/flink/conf/base-config.conf") - kafka { - input.topic = {{ env_name }}.post.certificate.process.request - output.audit.topic = {{ env_name }}.telemetry.raw - output.failed.topic = {{ env_name }}.post.certificate.process.failed - groupId = {{ env_name }}-post-certificate-process-group - } - task { - consumer.parallelism = {{ post_certificate_processor_consumer_parallelism }} - parallelism = {{ post_certificate_processor_parallelism }} - } - lms-cassandra { - keyspace = "{{ middleware_course_keyspace }}" - enrollment.table = "{{ middleware_user_enrollment_table }}" - course_batch.table = "{{ middleware_course_batch_table }}" - } - learner-service { - basePath = "{{ learner_service_base_url }}" - } - - flink-conf: |+ - jobmanager.memory.flink.size: {{ flink_job_names['post-publish-processor'].jobmanager_memory }} - taskmanager.memory.flink.size: {{ flink_job_names['post-publish-processor'].taskmanager_memory }} - taskmanager.numberOfTaskSlots: {{ flink_job_names['post-publish-processor'].taskslots }} - parallelism.default: 1 - jobmanager.execution.failover-strategy: region - taskmanager.memory.network.fraction: 0.1 - -certificate-pre-processor: - certificate-pre-processor: |+ - include file("/data/flink/conf/base-config.conf") - kafka { - input.topic = {{ env_name }}.issue.certificate.request - groupId = {{ env_name }}-certificate-pre-processor-group - output.topic = {{ env_name }}.generate.certificate.request - output.failed.topic = {{ env_name }}.certificate.events.failed - } - task { - consumer.parallelism = {{ certificate_pre_processor_consumer_parallelism }} - parallelism = {{ certificate_pre_processor_parallelism }} - } - lms-cassandra { - keyspace = "{{ middleware_course_keyspace }}" - batchTable = "course_batch" - userTable = "user_enrolments" - assessmentAggregatorTable = "assessment_aggregator" - } - content { - search { - basePath = "{{ kp_search_service_base_url }}" - } - basePath = "{{ kp_content_service_base_url }}" - } - lms { - basePath = "{{ lms_service_base_url }}" - } - cert { - basePath = "{{ certificate_base_path }}" - } - learner-service { - basePath = "{{ learner_service_base_url }}" - } - - flink-conf: |+ - jobmanager.memory.flink.size: {{ flink_job_names['certificate-pre-processor'].jobmanager_memory }} - taskmanager.memory.flink.size: {{ flink_job_names['certificate-pre-processor'].taskmanager_memory }} - taskmanager.numberOfTaskSlots: {{ flink_job_names['certificate-pre-processor'].taskslots }} - parallelism.default: 1 - jobmanager.execution.failover-strategy: region - taskmanager.memory.network.fraction: 0.1 From 7c08992580dc333f35ef65bd5faf06d9c7729e0f Mon Sep 17 00:00:00 2001 From: Mahesh Kumar Gangula Date: Tue, 5 Jan 2021 22:23:42 +0530 Subject: [PATCH 2/6] Issue #000 fix: questionset publish job configuration. --- .../roles/flink-jobs-deploy/defaults/main.yml | 7 +++++ .../helm_charts/datapipeline_jobs/values.j2 | 30 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 4e867d1892..49cea5fa63 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -125,6 +125,13 @@ flink_job_names: taskmanager_memory: 1024m taskslots: 1 cpu_requests: 0.3 + questionset-publish: + job_class_name: 'org.sunbird.job.task.QuestionSetPublishStreamTask' + replica: 1 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskslots: 1 + cpu_requests: 0.3 ### Global vars middleware_course_keyspace: "sunbird_courses" diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 2e32435f92..ad83c73756 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -192,3 +192,33 @@ post-publish-processor: jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 +questionset-publish: + questionset-publish: |+ + include file("/data/flink/conf/base-config.conf") + kafka { + input.topic = {{ env_name }}.learning.job.request + post_publish.topic = {{ env_name }}.content.postpublish.request + groupId = {{ env_name }}-questionset-publish-group + } + task { + consumer.parallelism = 1 + parallelism = 1 + router.parallelism = 1 + } + question { + keyspace = "dev_question_store" + table = "question_data" + } + questionset { + keyspace = "dev_hierarchy_store" + table = "questionset_hierarchy" + } + + flink-conf: |+ + jobmanager.memory.flink.size: {{ flink_job_names['questionset-publish'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['questionset-publish'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['questionset-publish'].taskslots }} + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + From 845a3a53c39892ca7a3c440d1411f772c9075892 Mon Sep 17 00:00:00 2001 From: Mahesh Kumar Gangula Date: Mon, 11 Jan 2021 11:56:00 +0530 Subject: [PATCH 3/6] Issue #000 fix: assessment.publish.request - topic creation. --- ansible/roles/setup-kafka/defaults/main.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ansible/roles/setup-kafka/defaults/main.yml b/ansible/roles/setup-kafka/defaults/main.yml index cf970c394f..7c7ac5f9fd 100644 --- a/ansible/roles/setup-kafka/defaults/main.yml +++ b/ansible/roles/setup-kafka/defaults/main.yml @@ -117,6 +117,9 @@ processing_kafka_topics: - name: generate.certificate.failed num_of_partitions: 1 replication_factor: 1 + - name: assessment.publish.request + num_of_partitions: 1 + replication_factor: 1 processing_kafka_overriden_topics: - name: telemetry.raw @@ -228,5 +231,6 @@ processing_kafka_overriden_topics: - name: generate.certificate.failed retention_time: 1209600000 replication_factor: 1 - - + - name: assessment.publish.request + retention_time: 1209600000 + replication_factor: 1 From 1e6b867edef49e357d38b785d5c147e9b1c3d9f6 Mon Sep 17 00:00:00 2001 From: Mahesh Kumar Gangula Date: Mon, 11 Jan 2021 13:03:39 +0530 Subject: [PATCH 4/6] Issue #000 fix: input topic name changes. --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index ad83c73756..71d7a3288e 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -196,7 +196,7 @@ questionset-publish: questionset-publish: |+ include file("/data/flink/conf/base-config.conf") kafka { - input.topic = {{ env_name }}.learning.job.request + input.topic = {{ env_name }}.assessment.publish.request post_publish.topic = {{ env_name }}.content.postpublish.request groupId = {{ env_name }}-questionset-publish-group } From 4e2a6d92764dac4dc38c6c3d2495259d62ecff35 Mon Sep 17 00:00:00 2001 From: Mahesh Kumar Gangula Date: Mon, 11 Jan 2021 13:22:43 +0530 Subject: [PATCH 5/6] Issue #000 fix: keyspace variables. --- ansible/inventory/env/group_vars/all.yml | 1 + kubernetes/helm_charts/datapipeline_jobs/values.j2 | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml index cc396d3d41..f08cd4638d 100644 --- a/ansible/inventory/env/group_vars/all.yml +++ b/ansible/inventory/env/group_vars/all.yml @@ -21,6 +21,7 @@ hierarchy_keyspace_name: "{{instance}}_hierarchy_store" orchestrator_keyspace_name: "{{instance}}_script_store" dialcode_keyspace_name: "{{instance}}_dialcode_store" category_keyspace_name: "{{instance}}_category_store" +assessment_keyspace_name: "{{instance}}_question_store" dp_cassandra_connection: "{{ groups['dp-cassandra'][0] }}:9042" diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 71d7a3288e..9a810e93b2 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -206,11 +206,11 @@ questionset-publish: router.parallelism = 1 } question { - keyspace = "dev_question_store" + keyspace = "{{ assessment_keyspace_name }}" table = "question_data" } questionset { - keyspace = "dev_hierarchy_store" + keyspace = "{{ hierarchy_keyspace_name }}" table = "questionset_hierarchy" } From d8a069580df88e4a304795591342193948205ecc Mon Sep 17 00:00:00 2001 From: Mahesh Kumar Gangula Date: Mon, 11 Jan 2021 15:44:00 +0530 Subject: [PATCH 6/6] Issue #000 fix: remove script store configurations. --- ansible/inventory/env/group_vars/all.yml | 1 - .../cassandra-db-update/templates/data.cql.j2 | 17 ----------------- .../templates/application.conf.j2 | 2 -- .../src/test/resources/application.conf | 2 -- .../service/src/main/resources/application.conf | 2 -- 5 files changed, 24 deletions(-) diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml index f08cd4638d..22a49038fa 100644 --- a/ansible/inventory/env/group_vars/all.yml +++ b/ansible/inventory/env/group_vars/all.yml @@ -18,7 +18,6 @@ learner_user_home: "/home/{{ learner_user }}" # these variables can be defined as {{instance}}_* content_keyspace_name: "{{instance}}_content_store" hierarchy_keyspace_name: "{{instance}}_hierarchy_store" -orchestrator_keyspace_name: "{{instance}}_script_store" dialcode_keyspace_name: "{{instance}}_dialcode_store" category_keyspace_name: "{{instance}}_category_store" assessment_keyspace_name: "{{instance}}_question_store" diff --git a/ansible/roles/cassandra-db-update/templates/data.cql.j2 b/ansible/roles/cassandra-db-update/templates/data.cql.j2 index 22d799af0b..ea78687892 100644 --- a/ansible/roles/cassandra-db-update/templates/data.cql.j2 +++ b/ansible/roles/cassandra-db-update/templates/data.cql.j2 @@ -13,18 +13,6 @@ CREATE TABLE IF NOT EXISTS {{ content_keyspace_name }}.content_data ( PRIMARY KEY (content_id) ); -CREATE KEYSPACE IF NOT EXISTS {{ orchestrator_keyspace_name }} WITH replication = { - 'class': 'SimpleStrategy', - 'replication_factor': '1' -}; - -CREATE TABLE IF NOT EXISTS {{ orchestrator_keyspace_name }}.script_data ( - name text, - type text, - reqmap text, - PRIMARY KEY (name) -); - CREATE TABLE IF NOT EXISTS {{ content_keyspace_name }}.question_data ( question_id text, last_updated_on timestamp, @@ -65,11 +53,6 @@ ALTER KEYSPACE {{ hierarchy_keyspace_name }} WITH replication = { 'datacenter1' : 2 }; -ALTER KEYSPACE {{ orchestrator_keyspace_name }} WITH replication = { - 'class': 'NetworkTopologyStrategy', - 'datacenter1' : 2 -}; - ALTER TABLE {{ content_keyspace_name }}.content_data ADD (externallink text); {% endif %} diff --git a/ansible/roles/learning-service/templates/application.conf.j2 b/ansible/roles/learning-service/templates/application.conf.j2 index 52dcda1ffa..7aa0a428b6 100644 --- a/ansible/roles/learning-service/templates/application.conf.j2 +++ b/ansible/roles/learning-service/templates/application.conf.j2 @@ -14,8 +14,6 @@ redis.port={{ redis_port }} redis.maxConnections=128 # Cassandra Configuration -orchestrator.keyspace.name="{{ orchestrator_keyspace_name }}" -orchestrator.keyspace.table="script_data" content.keyspace.name="{{ content_keyspace_name }}" hierarchy.keyspace.name="{{ hierarchy_keyspace_name }}" content.hierarchy.table="content_hierarchy" diff --git a/platform-core/unit-tests/src/test/resources/application.conf b/platform-core/unit-tests/src/test/resources/application.conf index e3f166efa4..169b02f75e 100644 --- a/platform-core/unit-tests/src/test/resources/application.conf +++ b/platform-core/unit-tests/src/test/resources/application.conf @@ -21,8 +21,6 @@ content.metadata.visibility.parent=["textbookunit", "courseunit", "lessonplanuni # Cassandra Configuration content.keyspace.name=content_store_test -orchestrator.keyspace.name="script_store" -orchestrator.keyspace.table="script_data" hierarchy.keyspace.name=hierarchy_store_test content.hierarchy.table=content_hierarchy_test framework.hierarchy.table=framework_hierarchy_test diff --git a/platform-modules/service/src/main/resources/application.conf b/platform-modules/service/src/main/resources/application.conf index 76bad61057..065c2c1c92 100644 --- a/platform-modules/service/src/main/resources/application.conf +++ b/platform-modules/service/src/main/resources/application.conf @@ -21,8 +21,6 @@ content.metadata.visibility.parent=["textbookunit", "courseunit", "lessonplanuni # Cassandra Configuration content.keyspace.name=content_store -orchestrator.keyspace.name=script_store -orchestrator.keyspace.table=script_data cassandra.lp.connection="127.0.0.1:9042,127.0.0.2:9042,127.0.0.3:9042" cassandra.lpa.connection="127.0.0.1:9042,127.0.0.2:9042,127.0.0.3:9042"