merge hadoop streaming code

.travis.yml

@@ -0,0 +1,34 @@
+language: java
+jdk:
+  - oraclejdk7
+before_install:
+    - sudo apt-get install mysql-server -y
+    - sudo apt-get install curl -y
+    - sudo service mysql restart
+    - echo hibernate.current_session_context_class=thread > md-dao/src/main/resources/db.properties
+    - echo hibernate.transaction.factory_class=org.hibernate.transaction.JDBCTransactionFactory >> md-dao/src/main/resources/db.properties
+    - echo hibernate.show_sql=false >> md-dao/src/main/resources/db.properties
+    - echo hibernate.connection.driver_class=com.mysql.jdbc.Driver >> md-dao/src/main/resources/db.properties
+    - echo "hibernate.connection.url=jdbc:mysql://localhost:3306/bdre?createDatabaseIfNotExist=true" >> md-dao/src/main/resources/db.properties
+    - echo hibernate.connection.password= >> md-dao/src/main/resources/db.properties
+    - echo hibernate.connection.username=root >> md-dao/src/main/resources/db.properties
+    - echo hibernate.dialect=org.hibernate.dialect.MySQLDialect >> md-dao/src/main/resources/db.properties
+    - echo hibernate.default_schema=bdre >> md-dao/src/main/resources/db.properties
+    - cat md-dao/src/main/resources/db.properties
+    - mysql -uroot -e "create database if not exists bdre"
+    - mysql -uroot bdre < databases/mysql/ddls/drop_tables.sql
+    - mysql -uroot bdre < databases/mysql/ddls/create_tables.sql
+    - wget http://www.us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.zip
+    - unzip apache-maven-3.3.9-bin.zip
+    - export PATH=$PWD/apache-maven-3.3.9/bin:$PATH
+    - export M2_HOME=$PWD/apache-maven-3.3.9
+install: /bin/true
+script:
+    - mvn -s settings.xml clean install -P hdp22 -pl '!im-crawler' --quiet
+    - sh install-scripts.sh local
+    - sudo service bdre start
+after_success:
+    - echo "Looks like success !"
+after_failure:
+    - cat /var/log/BDRE/daemon.log
+

README.md

@@ -1,24 +1,24 @@
 # Bigdata Ready Enterprise Open Source Software
 ## Table of Contents
 
-[License](#license)  
-[Objective](#objective)  
-[Features](#features)  
-[Architecture](#architecture)  
-[Installation](#installation)  
-[Demo videos](#data-ingestion)  
-       [Data Ingestion](#data-ingestion)  
-       [Workflow Builder](#workflow-builder)  
-       [Bulk Data Manufacturing](#bulk-data-manufacturing)  
-       [Web Crawler](#web-crawler)  
-[Operational Metadata Management](#operational-metadata-management)  
-[How To Contribute](#how-to-contribute)  
+[License](#license)
+[Objective](#objective)
+[Features](#features)
+[Demo videos](#demo-videos)
+       [Data Ingestion](#data-ingestion)
+       [Workflow Builder](#workflow-builder)
+       [Bulk Data Manufacturing](#bulk-data-manufacturing)
+       [Web Crawler](#web-crawler)
+[Architecture](#architecture)
+[Installation](#installation)
+[Operational Metadata Management System](#operational-metadata-management-system)
+[How To Contribute](#how-to-contribute)
 
 # License
 Released under Apache Public License 2.0. You can get a copy of the license at http://www.apache.org/licenses/LICENSE-2.0.
 # Objective
 Big Data Ready Enterprise(BDRE) makes big data technology adoption simpler by optimizing and integrating various big data solutions and providing them under one integrated package. BDRE provides a unified framework for a Hadoop implementation that can drastically minimize development time and fast track the Hadoop implementation. It comprises a reusable framework that can be customized as per the enterprise ecosystem. The components are loosely integrated and can be de-coupled or replaced easily with alternatives.
- 
+
 The primary goal of BDRE is to accelerate Bigdata implementations by supplying the essential frameworks that are most likely to be written from scratch. It can drastically reduce effort by eliminating hundreds of man hours in operational framework development. Big Data implementations however, require specialized skills, significant development effort on data loading, semantic processing, DQ, code deployment across environments etc.
 
 # Features
@@ -53,6 +53,38 @@ The primary goal of BDRE is to accelerate Bigdata implementations by supplying t
  - Create ingestion, data generation, Crawler jobs or create Oozie workflows graphically without writing any code
  - One click deploy and execute jobs without SSH into the edge node.
 
+# Demo Videos
+## Data Ingestion
+
+### RDBMS Data Ingestion
+
+<a href="http://www.youtube.com/watch?v=JcbYU7oEmxc" target="_blank"><img src="http://wiproopensourcepractice.github.io/openbdre/bdreimages/rdbms.PNG"
+alt="BDRE RDBMS data ingestion demo video" width="240" height="180" border="10" /></a>
+
+
+### Streaming Data Ingestion
+
+<a href="http://www.youtube.com/watch?v=1yqoAVENrjo" target="_blank"><img src="http://wiproopensourcepractice.github.io/openbdre/bdreimages/twitter.PNG"
+alt="BDRE Twitter Ingestion demo video" width="240" height="180" border="10" /></a>
+
+
+### Directory Monitoring and File Ingestion
+
+<a href="http://www.youtube.com/watch?v=IhDMYase1fU" target="_blank"><img src="http://wiproopensourcepractice.github.io/openbdre/bdreimages/filemon.PNG"
+alt="BDRE File ingestion demo video" width="240" height="180" border="10" /></a>
+
+## Workflow Builder
+
+<a href="http://www.youtube.com/watch?v=PG6Qvg-pKO0" target="_blank"><img src="http://wiproopensourcepractice.github.io/openbdre/bdreimages/wfd.PNG"
+alt="BDRE Workflow Designer demo video" width="240" height="180" border="10" /></a>
+
+## Bulk Data Manufacturing
+Demo video TBD
+## Web Crawler
+
+<a href="http://www.youtube.com/watch?v=0b6dWGxin4Y" target="_blank"><img src="http://wiproopensourcepractice.github.io/openbdre/bdreimages/crawler.PNG"
+alt="BDRE Web Crawling" width="240" height="180" border="10" /></a>
+
 # Architecture
 
 ![image](http://wiproopensourcepractice.github.io/openbdre/bdreimages/architecture.PNG)
@@ -186,9 +218,10 @@ You should be able to do the same in Mac or Windows but note that setting up a H
     ```shell
     mvn -s settings.xml clean install -P hdp22
     ```
- * *Note:* Selecting hdp22 will compile BDRE with HDP 2.2 libraries and automatically configure BDRE with Hortonworks Sandbox 2.2.0. Similarly one should be able to build this using -P cdh52 which will configure BDRE for CDH 5.2 QuickStart VM. During building it'll pick up the environment specific configurations from <source root>/databases/setup/profile.*hdp22*.properties.
+ * *Note:* Selecting hdp22 will compile BDRE with HDP 2.2 libraries and automatically configure BDRE with properties from  databases/setup/profile.hdp22.properties . These properties can later be altered from the BDRE Settings page under Administration.
+
+    databases/setup/profile.hdp22.properties looks like this.
 
-    Content of databases/setup/profile.hdp22.properties
  ```properties
     bdre_user_name=openbdre
     name_node_hostname=sandbox.hortonworks.com
@@ -204,6 +237,11 @@ You should be able to do the same in Mac or Windows but note that setting up a H
     hive_jdbc_password=openbdre
  ```
 
+| Building BDRE for Cloudera QuickStart VM |
+| ------------- |
+| Similarly one should be able to build this using *-P cdh52* which will configure BDRE for CDH 5.2 QuickStart VM. During building it'll pick up the environment specific configurations from <source root>/databases/setup/profile.*cdh52*.properties. BDRE virtually works with any Hadoop distribution including IBM's BigInsight platform in Bluemix|
+
+
     ```shell
     $ mvn -s settings.xml clean install -P hdp22
     [INFO] Scanning for projects...
@@ -260,42 +298,8 @@ You should be able to do the same in Mac or Windows but note that setting up a H
 * Check the process in Oozie console *http://VM_IP:11000/oozie*
 * When the import job is complete start the *data load job*.
 
-# Data Ingestion
-
-## RDBMS Data Ingestion
-
-<a href="http://www.youtube.com/watch?v=JcbYU7oEmxc" target="_blank"><img src="http://wiproopensourcepractice.github.io/openbdre/bdreimages/rdbms.PNG" 
-alt="BDRE RDBMS data ingestion demo video" width="240" height="180" border="10" /></a>
-
-
-## Streaming Data Ingestion
-
-<a href="http://www.youtube.com/watch?v=1yqoAVENrjo" target="_blank"><img src="http://wiproopensourcepractice.github.io/openbdre/bdreimages/twitter.PNG" 
-alt="BDRE Twitter Ingestion demo video" width="240" height="180" border="10" /></a>
-
-
-## Directory Monitoring and File Ingestion
-
-<a href="http://www.youtube.com/watch?v=IhDMYase1fU" target="_blank"><img src="http://wiproopensourcepractice.github.io/openbdre/bdreimages/filemon.PNG" 
-alt="BDRE File ingestion demo video" width="240" height="180" border="10" /></a>
-
-# Workflow Builder
-
-<a href="http://www.youtube.com/watch?v=PG6Qvg-pKO0" target="_blank"><img src="http://wiproopensourcepractice.github.io/openbdre/bdreimages/wfd.PNG" 
-alt="BDRE Workflow Designer demo video" width="240" height="180" border="10" /></a>
-
-# Bulk Data Manufacturing
-Demo video TBD
-# Web Crawler
-
-<a href="http://www.youtube.com/watch?v=0b6dWGxin4Y" target="_blank"><img src="http://wiproopensourcepractice.github.io/openbdre/bdreimages/crawler.PNG" 
-alt="BDRE Web Crawling" width="240" height="180" border="10" /></a>
-
-
 # Operational Metadata Management System
 
-### Operational Metadata Management
-
 BDRE provides complete job/operational metadata management solution for Hadoop. At its core acts as a registry and tracker for different types of jobs running in different Hadoop clusters or as a standalone. It provides APIs to integrate with virtually any jobs.
 
 
@@ -313,52 +317,51 @@ Contribution for the enhancements in BDRE are welcome and humbly requested by us
 <a class="buttons github" href="https://github.com/WiproOpenSourcePractice/openbdre">Goto BDRE@GitHub</a>
 
  - Clone the main repo (if you havn't done already)
- 
+
 ```shell
-git clone "https://github.com/WiproOpenSourcePractice/openbdre.git" 
+git clone "https://github.com/WiproOpenSourcePractice/openbdre.git"
 cd openbdre
 ```
 
  - Add your forked repo where you have write access and create your own branch.
- 
+
 ```shell
 git remote add myrepo https://<your id>:<your password>@github.com/<YOUR ACCT NAME>/openbdre.git
 git checkout -b mybranch
 ```
 
  - Make and commit your changes to your own branch.
- 
+
 ```shell
 git commit -am "My changes"
 ```
 
  - Push to your own branch in your own remote repo (myrepo).
- 
+
 ```shell
 git push myrepo mybranch
 ```
 
  - Everyday better pull from the main repo(origin) and sync your repo with it.
- 
+
 ```shell
 git checkout develop
-git pull origin develop 
+git pull origin develop
 ```
 
  - Keep the develop branch only to have the latest main repo content. Make changes while you are in your own branch.
 
  - Sync your code with the main repo. Push the latest content pulled from the main repo to your own repo in your own branch.
 
 ```shell
-git checkout mybranch 
+git checkout mybranch
 git merge develop
 git push myrepo mybranch
 ```
 
  - When you are ready to submit your contribution to the main repo, please open a [pull request](https://help.github.com/articles/creating-a-pull-request).
  - Please join the community https://groups.google.com/forum/#!forum/bdre. If you have any questions/suggestions please email to [email protected] .
  - If you want to report a bug, see/request a feature or work on something. Please sign up at https://openbdre.atlassian.net
-
 
 
-[![Analytics](https://ga-beacon.appspot.com/UA-72345517-2/openbdre/README.md)](http://wiproopensourcepractice.github.io/openbdre/bdreimages/badge.svg)
+[![Analytics](https://ga-beacon.appspot.com/UA-72345517-2/openbdre/README.md)](http://wiproopensourcepractice.github.io/openbdre/bdreimages/badge.svg)

appstore-context.xml

@@ -0,0 +1,5 @@
+<!DOCTYPE Configure PUBLIC "-//Mort Bay Consulting//DTD Configure//EN" "http://jetty.mortbay.org/configure.dtd">
+<Configure class="org.eclipse.jetty.webapp.WebAppContext">
+    <Set name="contextPath">/store</Set>
+    <Set name="war"><SystemProperty name="user.home"/>/bdreappstore</Set>
+</Configure>

appstore.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+BDRE_HOME=~/bdre
+. $BDRE_HOME/bdre-scripts/env.properties
+BDRE_APPSTORE_REPO=~/bdreappstore
+BDRE_APPSTORE_APPS=~/bdreappstore-apps
+rm -f -r $BDRE_APPSTORE_APPS
+mkdir -p $BDRE_APPSTORE_APPS
+#Pull latest apps from the repo
+if [ -d "$BDRE_APPSTORE_REPO" ]; then
+    echo "refresing repo"
+    cd $BDRE_APPSTORE_REPO
+    git pull $repoName $branchName
+    if [ $? -ne 0 ]
+        then exit 1
+    fi
+else
+    echo "cloning repo for first time"
+    cd ~
+    git clone $gitURL
+    if [ $? -ne 0 ]
+        then exit 1
+    fi
+    cd $BDRE_APPSTORE_REPO
+fi
+for d in */ ; do
+    echo "checking $d related apps"
+    cd "$d"
+    for sd in */ ; do
+        echo "archiving $sd app"
+        cd "$sd"
+        zip -r -X ../"${sd%?}" *
+        if [ $? -ne 0 ]
+            then exit 1
+        fi
+        echo "archiving done"
+        cd ..
+    done
+    mkdir -p $BDRE_APPSTORE_APPS/"$d"
+    rm -f $BDRE_APPSTORE_APPS/"$d"/*.zip
+    mv ./*.zip $BDRE_APPSTORE_APPS/"$d"
+    cd ../
+done

auth-rest-api/src/main/java/com/wipro/ats/bdre/md/auth/RestWrapper.java

@@ -14,6 +14,7 @@
 
 package com.wipro.ats.bdre.md.auth;
 
+import org.apache.log4j.Logger;
 import org.codehaus.jackson.annotate.JsonProperty;
 
 import java.lang.reflect.InvocationTargetException;
@@ -32,18 +33,8 @@ public class RestWrapper {
     public static final String ERROR = "ERROR";
     private String result;
     private Object records;
-
-    @JsonProperty("Message")
-    public String getMessage() {
-        return message;
-    }
-
-    public void setMessage(String message) {
-        this.message = message;
-    }
-
     private String message;
-
+    private static final Logger LOGGER = Logger.getLogger(RestWrapper.class);
 
     public RestWrapper(Object objectToSerialize, String result) {
         this.result = result;
@@ -54,6 +45,15 @@ public RestWrapper(Object objectToSerialize, String result) {
         }
     }
 
+    @JsonProperty("Message")
+    public String getMessage() {
+        return message;
+    }
+
+    public void setMessage(String message) {
+        this.message = message;
+    }
+
     @JsonProperty("Result")
     public String getResult() {
         return result;
@@ -83,26 +83,22 @@ public void setRecord(Object record) {
 
     @JsonProperty("TotalRecordCount")
     public int getTotalRecordCount() {
-        if (records != null) {
-            if (records instanceof Collection) {
+
+            if ( records != null && records instanceof Collection) {
                 Collection c = (Collection) records;
                 for (Object obj : c) {
                     try {
                         Method method = obj.getClass().getMethod("getCounter");
-                        Integer count = (Integer) method.invoke(obj);
-                        return count;
-                    } catch (NoSuchMethodException e) {
-                        return 0;
-                    } catch (InvocationTargetException e) {
-                        return 0;
-                    } catch (IllegalAccessException e) {
+                        return (Integer) method.invoke(obj);
+                    } catch (Exception e) {
+                        LOGGER.info(e);
                         return 0;
                     }
                 }
             } else {
                 return 1;
             }
-        }
+
         return 0;
     }
 

auth-rest-api/src/main/java/com/wipro/ats/bdre/md/auth/SessionHolder.java

@@ -26,15 +26,17 @@
  * Created by AR288503 on 9/6/2015.
  */
 public class SessionHolder {
+
     public static final int MAX_INACTIVE_SESSION_DURATION = 600;
+    private static Map<String, AuthResult> authResultMap = new HashMap<String, AuthResult>();
+
+    private SessionHolder(){}
 
     public static void resetSessionTimeout(HttpSession session) {
         int timeElapsed = (int) (new Date().getTime() - session.getCreationTime()) / 1000;
         session.setMaxInactiveInterval(SessionHolder.MAX_INACTIVE_SESSION_DURATION + timeElapsed);
     }
 
-    private static Map<String, AuthResult> authResultMap = new HashMap<String, AuthResult>();
-
     public static Map<String, AuthResult> getAuthResultMap() {
         return authResultMap;
     }