Deploy merge test

ansible/inventory/env/group_vars/all.yml

@@ -108,7 +108,7 @@ cert_rc_base_url: "http://{{private_ingressgateway_ip}}/registry-service/api/v1"
 cert_rc_entity: "TrainingCertificate"
 certificate_base_path: "https://{{domain_name}}/certs"
 cert_domain_url:  "{{proto}}://{{domain_name}}"
-cert_container_name: "{{cert_env_prefix}}-e-credentials"
+cert_container_name: "e-credentials"
 cert_cloud_storage_type: "azure"
 cert_azure_storage_secret: "{{sunbird_private_storage_account_key}}"
 cert_azure_storage_key: "{{sunbird_private_storage_account_name}}"

ansible/roles/learning-service/templates/application.conf.j2

@@ -196,9 +196,7 @@ job.request.event.mimetype=["application/pdf",
                               "image/x-quicktime",
                               "video/avi",
                               "video/mpeg",
-                              "video/quicktime",
                               "video/3gpp",
-                              "video/mp4",
                               "video/ogg",
                               "video/webm",
                               "video/msvideo",
@@ -209,7 +207,6 @@ job.request.event.mimetype=["application/pdf",
                               "audio/mp4",
                               "audio/mpeg",
                               "audio/ogg",
-                              "audio/webm",
                               "audio/x-wav",
                               "audio/wav",
                               "audio/mpeg3",
@@ -253,12 +250,10 @@ cloud_storage_type="{{ cloud_store }}"
 azure_storage_key="{{sunbird_public_storage_account_name}}"
 azure_storage_secret="{{sunbird_public_storage_account_key}}"
 azure_storage_container="{{ azure_public_container }}"
-
 aws_storage_key = "{{ aws_storage_key }}"
 aws_storage_secret: "{{ aws_storage_secret }}"
 aws_storage_container: "{{ aws_storage_container }}"
 
-
 installation.id="{{ instance_name }}"
 
 # Configuration for DIALCode linking

ansible/roles/lp-contenttool/templates/application.conf.j2

@@ -10,6 +10,9 @@ destination.storage_type="{{ contenttool.destination.storage_type }}"
 azure_storage_key="{{sunbird_public_storage_account_name}}"
 azure_storage_secret="{{sunbird_public_storage_account_key}}"
 azure_storage_container="{{ azure_public_container }}"
+aws_storage_key = "{{ aws_storage_key }}"
+aws_storage_secret: "{{ aws_storage_secret }}"
+aws_storage_container: "{{ aws_storage_container }}"
 
 content.external_fields="body,stageIcons,screenshots"
 content.extract_mimetype="application/vnd.ekstep.h5p-archive,application/vnd.ekstep.ecml-archive,application/vnd.ekstep.html-archive"

ansible/roles/lp-synctool-deploy/defaults/main.yml

@@ -23,6 +23,6 @@ redis_port: 6379
 graphevent_kafka_url: "{{groups['processing-cluster-zookeepers']|join(':9092,')}}:9092"
 search_lms_index_host: "{{ groups['core-es']|join(':9200,')}}:9200"
 
-cloud_store: azure
+cloud_store: aws
 azure_public_container: 
 azure_account_name:

ansible/roles/lp-synctool-deploy/templates/application.conf.j2

@@ -81,6 +81,9 @@ cloud_storage_type="{{ cloud_store }}"
 azure_storage_key="{{sunbird_public_storage_account_name}}"
 azure_storage_secret="{{sunbird_public_storage_account_key}}"
 azure_storage_container="{{ azure_public_container }}"
+aws_storage_key = "{{ aws_storage_key }}"
+aws_storage_secret: "{{ aws_storage_secret }}"
+aws_storage_container: "{{ aws_storage_container }}"
 
 contentTypeToPrimaryCategory {
   ClassroomTeachingVideo: "Explanation Content"

ansible/roles/samza-jobs/tasks/deploy.yml

@@ -71,7 +71,17 @@
   replace: dest="{{item[1].stdout}}" regexp="{{item[0].key}}" replace="{{item[0].value}}"
   when: "{{item[1]|changed}}"
   with_nested:
-    - [{key: "__yarn_host__", value: "{{__yarn_host__}}"}, {key: "__yarn_port__", value: "{{__yarn_port__}}"}, {key: "__env__", value: "{{env}}" }, {key: "__env_name__", value: "{{env_name}}" }, {key: "__zookeepers__", value: "{{zookeepers}}"}, {key: "__kafka_brokers__", value: "{{kafka_brokers}}"}, {key: "__delayInMilliSeconds__", value: "{{delayInMilliSeconds}}" }, {key: "__retryTimeInMilliSeconds__", value: "{{retryTimeInMilliSeconds}}" }, {key: "__bypass_reverse_search__", value: "{{bypass_reverse_search}}" }, {key: "__retryBackoffBaseInSeconds__", value: "{{retry_backoff_base_in_seconds}}" }, {key: "__retryLimit__", value: "{{retry_limit}}" }, {key: "__retryLimitEnable__", value: "{{retry_limit_enable}}" },  {key: "__google_api_key__", value: "{{google_api_key}}" }, {key: "__searchServiceEndpoint__", value: "{{search_service_endpoint}}" }, {key: "__objectDenormalizationAdditionalConfig__", value: "{{object_denormalization_additional_config}}" },{key: "__audit_es_host__", value: "{{audit_es_host}}"}, {key: "__search_es_host__", value: "{{search_es_host}}"}, {key: "__redis_host__", value: "{{redis_host}}"}, {key: "__dp_redis_host__", value: "{{dp_redis_host}}"}, {key: "__redis_port__", value: "{{redis_port}}"}, {key: "__environment_id__", value: "{{environment_id}}"}, {key: "__graph_passport_key__", value: "{{graph_passport_key}}"}, {key: "__lp_bolt_url__", value: "{{lp_bolt_url}}"}, {key: "__lp_bolt_read_url__", value: "{{lp_bolt_read_url}}"}, {key: "__lp_bolt_write_url__", value: "{{lp_bolt_write_url}}"}, {key: "__other_bolt_url__", value: "{{other_bolt_url}}"}, {key: "__other_bolt_read_url__", value: "{{other_bolt_read_url}}"}, {key: "__other_bolt_write_url__", value: "{{other_bolt_write_url}}"}, {key: "__mw_shard_id__", value: "{{mw_shard_id}}"}, {key: "__lp_url__", value: "{{lp_url}}"}, {key: "__cloud_storage_config_environment__", value: "{{cloud_storage_config_environment}}"}, {key: "__google_vision_tagging__", value: "{{google_vision_tagging}}"}, {key: "__lp_tmpfile_location__", value: "{{lp_tmpfile_location}}"}, {key: "__esRouterAdditionalConfig__", value: "{{es_router_additional_config}}"},{key: "__esRouterSecondaryAdditionalConfig__", value: "{{es_router_additional_secondary_config}}"},{key: "__es_port__", value: "{{es_port}}"}, {key: "__keyspace_name__", value: "{{content_keyspace_name}}"}, {key: "__collection_fullecar_disable__", value: "{{collection_fullecar_disable}}"},{key: "__max_iteration_count_for_samza_job__", value: "{{max_iteration_count_for_samza_job}}"},{key: "__cloud_storage_type__", value: "{{cloud_store}}"},{key: "__azure_storage_key__", value: "{{sunbird_public_storage_account_name}}"},{key: "__azure_storage_secret__", value: "{{sunbird_public_storage_account_key}}"},{key: "__azure_storage_container__", value: "{{azure_public_container}}"},{key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__plugin_media_base_url__", value: "{{plugin_media_base_url}}"}, {key: "__installation_id__", value: "{{instance_name}}"}, {key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__hierarchy_keyspace_name__", value: "{{hierarchy_keyspace_name}}"},  {key: "__composite_search_indexer_container_count__", value: "{{composite_search_indexer_container_count}}"},{key: "__cassandra_lp_connection__", value: "{{lp_cassandra_connection}}"}, {key: "__cassandra_lpa_connection__", value: "{{dp_cassandra_connection}}"}, {key: "__streaming_mime_type__", value: "{{streaming_mime_type}}"}, {key: "__cassandra_sunbird_connection__", value: "{{core_cassandra_connection}}"}, {key: "__cloud_upload_retry_count__", value: "{{cloud_upload_retry_count}}"}, {key: "__compositesearch_index_name__", value: "{{compositesearch_index_name}}"},{key: "__publish_pipeline_container_count__", value: "{{publish_pipeline_container_count}}"},{key: "__yarn_container_memory_mb__", value: "{{publish_yarn_container_memory_mb}}"},{key: "__youtube_api_key__", value: "{{youtube_api_key}}"},{key: "__kp_learning_service_base_url__", value: "{{kp_learning_service_base_url}}"},{key: "__sunbird_installation__", value: "{{sunbird_platform_installation}}"},  {key: "__search_lms_es_host__", value: "{{search_lms_es_host}}"},{key: "__dial_image_storage_container__", value: "{{dial_image_storage_container}}"},{key: "__dial_base_url__", value: "{{dial_base_url}}"},{key: "__learner_service_base_url__", value: "{{learner_service_base_url}}"},{key: "__cert_service_base_url__", value: "{{cert_service_base_url}}"},{key: "__certificate_base_path__", value: "{{certificate_base_path}}"},{key: "__kp_content_service_base_url__", value: "{{kp_content_service_base_url}}"},{key: "__kp_print_service_base_url__", value: "{{kp_print_service_base_url}}"},{key: "__cert_reg_service_base_url__", value: "{{cert_reg_service_base_url}}"},{key: "__kp_search_service_base_url__", value: "{{kp_search_service_base_url}}"},{key: "__samza_coordinator_replication_factor__", value: "{{samza_coordinator_replication_factor}}"},{key: "__samza_checkpoint_replication_factor__", value: "{{samza_checkpoint_replication_factor}}"},{key: "__course_batch_updater_container_count__", value: "{{course_batch_updater_container_count}}"},{key: "__course_certificate_generator_container_count__", value: "{{course_certificate_generator_container_count}}"},{key: "__course_progress_batch_size__", value: "{{course_progress_batch_size}}"},{key: "__itemset_generate_pdf__", value: "{{itemset_generate_pdf}}"},{key: "__auto_creator_container_count__", value: "{{auto_creator_container_count}}"},{key: "__content_streaming_enabled__", value: "{{content_streaming_enabled}}"},{key: "__lms_service_base_url__", value: "{{lms_service_base_url}}"},{key: "__mvc_search_indexer_container_count__", value: "{{mvc_search_indexer_container_count}}"}, {key: "__search_es7_host__", value: "{{search_es7_host}}"} , {key: "__ml-keywordapi__", value: "{{mlworkbench}}"},{key: "__auto_creator_artifact_allowed_sources__", value: "{{auto_creator_artifact_allowed_sources}}"},{key: "__publish_pipeline_task_opts__", value: "{{publish_pipeline_task_opts}}"},{key: "__auto_creator_g_service_acct_cred__", value: "{{auto_creator_gservice_acct_cred}}"},{key: "__certificate_pre_processor_container_count__", value: "{{certificate_pre_processor_container_count}}"},{key: "__master_category_validation_enabled__", value: "{{master_category_validation_enabled}}"}]
+    - [{key: "__yarn_host__", value: "{{__yarn_host__}}"}, {key: "__yarn_port__", value: "{{__yarn_port__}}"}, {key: "__env__", value: "{{env}}" }, {key: "__env_name__", value: "{{env_name}}" }, {key: "__zookeepers__", value: "{{zookeepers}}"}, {key: "__kafka_brokers__", value: "{{kafka_brokers}}"}, {key: "__delayInMilliSeconds__", value: "{{delayInMilliSeconds}}" }, {key: "__retryTimeInMilliSeconds__", value: "{{retryTimeInMilliSeconds}}" }, {key: "__bypass_reverse_search__", value: "{{bypass_reverse_search}}" }, {key: "__retryBackoffBaseInSeconds__", value: "{{retry_backoff_base_in_seconds}}" }, {key: "__retryLimit__", value: "{{retry_limit}}" }, {key: "__retryLimitEnable__", value: "{{retry_limit_enable}}" },  {key: "__google_api_key__", value: "{{google_api_key}}" }, {key: "__searchServiceEndpoint__", value: "{{search_service_endpoint}}" }, {key: "__objectDenormalizationAdditionalConfig__", value: "{{object_denormalization_additional_config}}" },{key: "__audit_es_host__", value: "{{audit_es_host}}"}, {key: "__search_es_host__", value: "{{search_es_host}}"}, {key: "__redis_host__", value: "{{redis_host}}"}, {key: "__dp_redis_host__", value: "{{dp_redis_host}}"}, {key: "__redis_port__", value: "{{redis_port}}"}, {key: "__environment_id__", value: "{{environment_id}}"}, {key: "__graph_passport_key__", value: "{{graph_passport_key}}"}, {key: "__lp_bolt_url__", value: "{{lp_bolt_url}}"}, {key: "__lp_bolt_read_url__", value: "{{lp_bolt_read_url}}"}, {key: "__lp_bolt_write_url__", value: "{{lp_bolt_write_url}}"}, {key: "__other_bolt_url__", value: "{{other_bolt_url}}"}, {key: "__other_bolt_read_url__", value: "{{other_bolt_read_url}}"}, {key: "__other_bolt_write_url__", value: "{{other_bolt_write_url}}"}, {key: "__mw_shard_id__", value: "{{mw_shard_id}}"}, {key: "__lp_url__", value: "{{lp_url}}"}, {key: "__cloud_storage_config_environment__", value: "{{cloud_storage_config_environment}}"}, {key: "__google_vision_tagging__", value: "{{google_vision_tagging}}"}, {key: "__lp_tmpfile_location__", value: "{{lp_tmpfile_location}}"}, {key: "__esRouterAdditionalConfig__", value: "{{es_router_additional_config}}"},{key: "__esRouterSecondaryAdditionalConfig__", value: "{{es_router_additional_secondary_config}}"},{key: "__es_port__", value: "{{es_port}}"}, {key: "__keyspace_name__", value: "{{content_keyspace_name}}"}, {key: "__collection_fullecar_disable__", value: "{{collection_fullecar_disable}}"},{key: "__max_iteration_count_for_samza_job__", value: "{{max_iteration_count_for_samza_job}}"},{key: "__cloud_storage_type__", value: "{{cloud_store}}"},{key: "__aws_access_key_id__",value: "{{aws_storage_key}}"},{key: "__aws_secret_access_key__",value: "{{aws_storage_secret}}"},{key: "__aws_storage_container__",value: "{{aws_storage_container}}"},{key: "__azure_storage_key__", value: "{{sunbird_public_storage_account_name}}"},{key: "__azure_storage_secret__", value: "{{sunbird_public_storage_account_key}}"},{key: "__azure_storage_container__", value: "{{azure_public_container}}"},{key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__plugin_media_base_url__", value: "{{plugin_media_base_url}}"}, {key: "__installation_id__", value: "{{instance_name}}"}, {key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__hierarchy_keyspace_name__", value: "{{hierarchy_keyspace_name}}"},  {key: "__composite_search_indexer_container_count__", value: "{{composite_search_indexer_container_count}}"},{key: "__cassandra_lp_connection__", value: "{{lp_cassandra_connection}}"}, {key: "__cassandra_lpa_connection__", value: "{{dp_cassandra_connection}}"}, {key: "__streaming_mime_type__", value: "{{streaming_mime_type}}"}, {key: "__cassandra_sunbird_connection__", value: "{{core_cassandra_connection}}"}, {key: "__cloud_upload_retry_count__", value: "{{cloud_upload_retry_count}}"}, {key: "__compositesearch_index_name__", value: "{{compositesearch_index_name}}"},{key: "__publish_pipeline_container_count__", value: "{{publish_pipeline_container_count}}"},{key: "__yarn_container_memory_mb__", value: "{{publish_yarn_container_memory_mb}}"},{key: "__youtube_api_key__", value: "{{youtube_api_key}}"},{key: "__kp_learning_service_base_url__", value: "{{kp_learning_service_base_url}}"},{key: "__sunbird_installation__", value: "{{sunbird_platform_installation}}"},  {key: "__search_lms_es_host__", value: "{{search_lms_es_host}}"},{key: "__dial_image_storage_container__", value: "{{dial_image_storage_container}}"},{key: "__dial_base_url__", value: "{{dial_base_url}}"},{key: "__learner_service_base_url__", value: "{{learner_service_base_url}}"},{key: "__cert_service_base_url__", value: "{{cert_service_base_url}}"},{key: "__certificate_base_path__", value: "{{certificate_base_path}}"},{key: "__kp_content_service_base_url__", value: "{{kp_content_service_base_url}}"},{key: "__kp_print_service_base_url__", value: "{{kp_print_service_base_url}}"},{key: "__cert_reg_service_base_url__", value: "{{cert_reg_service_base_url}}"},{key: "__kp_search_service_base_url__", value: "{{kp_search_service_base_url}}"},{key: "__samza_coordinator_replication_factor__", value: "{{samza_coordinator_replication_factor}}"},{key: "__samza_checkpoint_replication_factor__", value: "{{samza_checkpoint_replication_factor}}"},{key: "__course_batch_updater_container_count__", value: "{{course_batch_updater_container_count}}"},{key: "__course_certificate_generator_container_count__", value: "{{course_certificate_generator_container_count}}"},{key: "__course_progress_batch_size__", value: "{{course_progress_batch_size}}"},{key: "__itemset_generate_pdf__", value: "{{itemset_generate_pdf}}"},{key: "__auto_creator_container_count__", value: "{{auto_creator_container_count}}"},{key: "__content_streaming_enabled__", value: "{{content_streaming_enabled}}"},{key: "__lms_service_base_url__", value: "{{lms_service_base_url}}"},{key: "__mvc_search_indexer_container_count__", value: "{{mvc_search_indexer_container_count}}"}, {key: "__search_es7_host__", value: "{{search_es7_host}}"} , {key: "__ml-keywordapi__", value: "{{mlworkbench}}"},{key: "__auto_creator_artifact_allowed_sources__", value: "{{auto_creator_artifact_allowed_sources}}"},{key: "__publish_pipeline_task_opts__", value: "{{publish_pipeline_task_opts}}"},{key: "__auto_creator_g_service_acct_cred__", value: "{{auto_creator_gservice_acct_cred}}"}]
+    - "{{ (config_files|default({})).results|default([]) }}"
+
+
+@@ -101,4 +101,4 @@
+  args:
+    chdir: "{{samza_jobs_dir}}/extract/"
+
+- file: path={{samza_jobs_dir}} owner=hduser group=hadoop state=directory recurse=yes
+- file: path={{samza_jobs_dir}} owner=hduser group=hadoop state=directory recurse=yes 
+
     - "{{ (config_files|default({})).results|default([]) }}"
 
 

docs/domain_model_v2/definitions/content_definition.json

@@ -271,7 +271,9 @@
             "audio/ogg",
             "audio/webm",
             "audio/x-wav",
-            "audio/wav"
+            "audio/wav",
+	    "application/json",
+	    "application/quiz"
           ],
           "defaultValue": "application/vnd.ekstep.ecml-archive",
           "renderingHints": "{ 'inputType': 'select',  'order': 9 }",

docs/domain_model_v2/definitions/content_image_definition.json

@@ -271,7 +271,9 @@
             "audio/ogg",
             "audio/webm",
             "audio/x-wav",
-            "audio/wav"
+            "audio/wav",
+            "application/json",
+            "application/quiz"
           ],
           "defaultValue": "application/vnd.ekstep.ecml-archive",
           "renderingHints": "{ 'inputType': 'select',  'order': 9 }",

kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml

@@ -39,17 +39,17 @@ producer_max_request_size: 1572864
 
 ### Activity Aggregate job related vars
 activity_agg_consumer_parallelism: 1
-activity_agg_dedup_parallelism: 1
-activity_agg_parallelism: 1
+activity_agg_dedup_parallelism: "{{ activity_aggregate_updater_replica | default('1') }}"
+activity_agg_parallelism: "{{ activity_aggregate_updater_replica | default('1') }}"
 enrolment_complete_parallelism: 1
 middleware_consumption_table: "user_content_consumption"
 middleware_user_activity_agg_table: "user_activity_agg"
-activity_agg_checkpointing_interval: 300000
-activity_agg_checkpointing_pause_interval: 90000
+activity_agg_checkpointing_interval: 180000
+activity_agg_checkpointing_pause_interval: 60000
 activity_agg_batch_interval: 60
 activity_agg_batch_read_size: 1
 activity_agg_batch_write_size: 10
-activity_agg_window_shards: 1000
+activity_agg_window_shards: 30
 activity_agg_dedup_index: 13
 activity_agg_dedup_expiry: 604800
 activity_module_aggs_enabled: true
@@ -140,7 +140,7 @@ flink_job_names:
     cpu_requests: 0.3
   activity-aggregate-updater:
     job_class_name: 'org.sunbird.job.aggregate.task.ActivityAggregateUpdaterStreamTask'
-    replica: 1
+    replica: "{{ activity_aggregate_updater_replica | default('1') }}"
     jobmanager_memory: 1024m
     taskmanager_memory: 1024m
     taskslots: 1

kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml

@@ -156,17 +156,20 @@ kind: Deployment
 metadata:
   name: {{ .Release.Name }}-taskmanager
   namespace: {{ .Values.namespace }}
+
 spec:
   replicas: {{ .Values.taskmanager.replicas }}
   selector:
     matchLabels:
       app: flink
       component: {{ .Release.Name }}-taskmanager
+
   template:
     metadata:
       labels:
         app: flink
         component: {{ .Release.Name }}-taskmanager
+
     spec:
       volumes:
       - name: flink-config-volume