Added file upload view with url pattern added to url.py

api/config.py

@@ -53,3 +53,5 @@
 #*********** Data Store ************************************************
 DATA_STORE_EXCLUDE = ['admin','local','cybercom_auth','system.users','cybercom_queue','df']
 DATA_STORE_MONGO_URI = 'mongodb://localhost:27017/'
+#*********** DOCKER_HOST_DATA_DIRECTORY ********************
+DOCKER_HOST_DATA_DIRECTORY = "/data"

api/settings.py

@@ -22,6 +22,12 @@
     "api.processor.title",
 )
 
+#DOCKER_HOST_DATA_DIRECTORY when host machine runs need to know if different then /data
+try
+    DOCKER_HOST_DATA_DIRECTORY= config.DOCKER_HOST_DATA_DIRECTORY
+except:
+    DOCKER_HOST_DATA_DIRECTORY="/data"
+
 # SECURITY WARNING: keep the secret key used in production secret!
 SECRET_KEY = config.SECRET_KEY
 
@@ -84,7 +90,8 @@
     'rest_framework.authtoken',
     'queue',
     'catalog',
-    'data_store'
+    'data_store',
+    'mgmic'
 
 )
 

api/urls.py

@@ -22,6 +22,7 @@
     url(r'^queue/', include('queue.urls')),
     url(r'^data_store/',include('data_store.urls')),
     url(r'^catalog/', include('catalog.urls')),
+    url(r'^mgmic/', include('mgmic.urls')),
     # Admin Urls
     url(r'^admin/', include(admin.site.urls)),
     # Main Project View - Customize depending on what Apps are enabled

api/views.py

@@ -61,6 +61,7 @@ def post(self,request,format=None):
         if password:
             user.set_password(password)
             data = {"password":"Successfully Updated"}
+	    user.save()
             return Response(data)
         auth_tok  = request.DATA.get('auth-token', None)
         if str(auth_tok).lower()=="update":

mgmic/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

mgmic/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

mgmic/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

mgmic/urls.py

@@ -0,0 +1,27 @@
+__author__ = 'mstacy'
+from django.conf.urls import patterns, url
+from django.contrib import admin
+#from queue.views import Run, Queue, UserTasks, UserResult,FileUploadView
+from rest_framework.urlpatterns import format_suffix_patterns
+from views import CheckMapFile
+# q = QueueTask()
+#tasks_url = []
+
+#for task in q.list()['available_tasks']:
+#    tasks_url.append(url(r'%s/$' % task, Run.as_view(), name="%s-run" % (task)))
+#    #tasks_url.append(url(r'%s/.(api|json|jsonp|xml|yaml)$' % task, Run.as_view(), name="%s-run-format" % (task)))
+
+admin.autodiscover()
+
+urlpatterns = patterns('',
+                       #url(r'run/', include(tasks_url)),
+                       #url(r'run/$',Run.as_view(),name='run-main'),
+                       url(r'check-mapfile/', CheckMapFile.as_view(), name='cmap-main'),
+		       #url(r'file-upload/', FileUploadView.as_view(), name='file-upload'),
+		       #url(r'file-upload/(?P<filename>[-\w.]+)', FileUploadView.as_view(), name='file-upload'),
+                       #url(r'task/(?P<task_id>[-\w]+)/$', UserResult.as_view(), name='queue-task-result'),
+                       #url(r'usertasks/$', UserTasks.as_view(), name='queue-user-tasks'),
+                       #url(r'^$', Queue.as_view(), name="queue-main"),
+)
+
+urlpatterns = format_suffix_patterns(urlpatterns, allowed=['api', 'json', 'jsonp', 'xml', 'yaml'])

mgmic/views.py

@@ -0,0 +1,131 @@
+from django.shortcuts import render
+#from rest_framework.settings import api_settings
+from rest_framework.response import Response
+from rest_framework.reverse import reverse
+from rest_framework.permissions import IsAuthenticatedOrReadOnly,DjangoModelPermissionsOrAnonReadOnly,AllowAny
+from rest_framework.views import APIView
+#from ccelery.q import QueueTask, list_tasks, task_docstring
+#from models import Run_model
+from rest_framework.renderers import JSONRenderer, JSONPRenderer
+#from renderer import QueueRunBrowsableAPIRenderer
+from rest_framework.parsers import JSONParser,MultiPartParser,FormParser,FileUploadParser
+#from util import trim
+#from rest_framework.authtoken.models import Token
+#from django.views.decorators.csrf import csrf_exempt
+#from django.utils.decorators import method_decorator
+#task = list_tasks()['available_tasks']
+#from rest_framework.viewsets import ModelViewSet
+#from serializer import FileUploadSerializer
+import os,glob,shutil,httplib
+from dockertask import docker_task
+from subprocess import Popen, PIPE, call
+from django.conf import settings
+from urlparse import urlparse
+import requests
+
+class CheckMapFile(APIView):
+    permission_classes = ( AllowAny,)
+    renderer_classes = (JSONRenderer,)
+    def __init__(self, *args, **kwargs):
+        self.docker_cmd = "validate_mapping_file.py -m %s -o %s "
+        temp = getattr(settings,"DOCKER_HOST_DATA_DIRECTORY", "/data")
+        self.docker_opts = "-v %s:/data" % (temp)
+        self.docker_container = "qiime_env"
+        super(CheckMapFile, self).__init__(*args, **kwargs)
+
+    def get_username(self, request):
+        username = "guest"
+        if request.user.is_authenticated():
+            username = request.user.username
+        return username
+
+    def post(self, request,format=None):
+        filename =request.POST.get('filename',None)
+        if filename:
+            if os.path.isfile(filename):
+                p = Popen(['md5sum', filename], stdin=PIPE, stdout=PIPE, stderr=PIPE)
+                output, err = p.communicate()
+                resultDir = os.path.join("/data/tmp", self.get_username(request),output.split()[0])
+                #pass
+            else:
+                if not check_url_exist(filename):
+                    raise Exception("Please Check URL or Local File Path(local files must be in /data directory) %s" % filename)
+                resultDir = os.path.join("/data/tmp", self.get_username(request))
+                map_read = os.path.join(resultDir,filename.split('/')[-1])
+                logfile= open(resultDir + "/logfile.txt","w")
+                call(['wget','-O',map_read,filename],stdout=logfile,stderr=logfile)
+                logfile.close()
+                p = Popen(['md5sum', map_read], stdin=PIPE, stdout=PIPE, stderr=PIPE)
+                output, err = p.communicate()
+                resultDir = os.path.join("/data/tmp", self.get_username(request),output.split()[0])
+                filename=map_read
+            try:
+                os.makedirs(resultDir)
+            except:
+                shutil.rmtree(resultDir)
+                os.makedirs(resultDir)
+        else:
+            raise Exception('Please provide map file filename')
+        docker_cmd = self.docker_cmd % (filename,resultDir)
+        result = docker_task(docker_name=self.docker_container,docker_opts=self.docker_opts,docker_command=docker_cmd,id=output.split()[0])
+        logfile= glob.glob(resultDir + '/*.log')
+        status = ""
+        with open(logfile[0]) as f:
+            data = f.read()
+            if "No errors" in data:
+                status="SUCCESS"
+            else:
+                status="FAILURE"
+        return Response({
+            'status':status,
+            'log': data,
+            'local-file': filename 
+        })
+
+def check_url_exist(url):
+    p = urlparse(url)
+    c = httplib.HTTPConnection(p.netloc)
+    c.request("HEAD", p.path)
+    return c.getresponse().status < 400
+# Create your views here.
+"""
+class FileUploadView(APIView):
+
+    permission_classes =(AllowAny,)
+    parser_classes = (MultiPartParser, FormParser,)
+    #parser_classes = (FileUploadParser,)
+    renderer_classes = (JSONRenderer,)
+
+    @method_decorator(csrf_exempt)
+    def dispatch(self, request, *args, **kwargs):
+        return super(FileUploadView, self).dispatch(request, *args, **kwargs)
+
+    def get_username(self, request):
+        username = "guest"
+        if request.user.is_authenticated():
+            username = request.user.username
+        return username
+
+    def post(self, request, format=None):
+    	resultDir = os.path.join("/data/tmp", self.get_username(request))
+	try:
+    	    os.makedirs(resultDir)
+	except:
+	    pass
+	result={}
+	for key,value in request.FILES.iteritems():
+	    filename= value.name
+	    local_file = "%s/%s" % (resultDir,filename)
+	    self.handle_file_upload(request.FILES[key],local_file)
+	    result[key]=local_file
+	return Response(result)
+    def handle_file_upload(self,f,filename):
+	if f.multiple_chunks():
+	    with open(filename, 'wb+') as temp_file:
+                for chunk in f.chunks():
+                    temp_file.write(chunk)
+	else:
+	    with open(filename, 'wb+') as temp_file:
+	        temp_file.write(f.read())
+
+"""

queue/ccelery/q.py

@@ -218,24 +218,43 @@ def result(self, task_id=None, redirect=True):
     def task(self, task_id=None):
         """Return task log and task results"""
         doc = self.db[self.database][self.collection].find_one({'task_id': task_id}, {'_id': False})
+	col = self.db[self.database][self.tomb_collection]
         if doc:
-            col = self.db[self.database][self.tomb_collection]
             result = col.find_one({'_id': task_id}, {'_id': False})
             if result:
-                if 'traceback' in result:
-                    result['traceback'] = pickle.loads(result['traceback'])
-                if 'children' in result:
-                    result['children'] = pickle.loads(result['children'])
-                if 'result' in result:
-                    result['result'] = pickle.loads(result['result'])
-                    if isinstance(result['result'], Exception):
-                        result['result'] = "ERROR: %s" % (result['result'].message)
+		result=self.unpickle_result(result)
+                #if 'traceback' in result:
+                #    result['traceback'] = pickle.loads(result['traceback'])
+                #if 'children' in result:
+                #    result['children'] = pickle.loads(result['children'])
+                #if 'result' in result:
+                #    result['result'] = pickle.loads(result['result'])
+                #    if isinstance(result['result'], Exception):
+                #        result['result'] = "ERROR: %s" % (result['result'].message)
             else:
                 result = self.status(task_id=task_id)
             doc['result'] = result
             return doc
         else:
-            raise Exception("Task was not found. Not a valid task_id")
+	    result = col.find_one({'_id': task_id}, {'_id': False})
+	    if result:
+		result=self.unpickle_result(result)
+		return result
+	    else:
+		result = self.status(task_id=task_id)
+		return result
+            	#raise Exception("Task was not found. Not a valid task_id")
+
+    def unpickle_result(self,result):
+	if 'traceback' in result:
+	    result['traceback'] = pickle.loads(result['traceback'])
+	if 'children' in result:
+	    result['children'] = pickle.loads(result['children'])
+	if 'result' in result:
+	    result['result'] = pickle.loads(result['result'])
+	    if isinstance(result['result'], Exception):
+		result['result'] = "ERROR: %s" % (result['result'].message)
+	return result
 
 
     def reset(self, user=None):

queue/urls.py

@@ -1,7 +1,7 @@
 __author__ = 'mstacy'
 from django.conf.urls import patterns, url
 from django.contrib import admin
-from queue.views import Run, Queue, UserTasks, UserResult
+from queue.views import Run, Queue, UserTasks, UserResult,FileUploadView
 from rest_framework.urlpatterns import format_suffix_patterns
 
 # q = QueueTask()
@@ -17,6 +17,8 @@
                        #url(r'run/', include(tasks_url)),
                        #url(r'run/$',Run.as_view(),name='run-main'),
                        url(r'run/(?P<task_name>[-\w .]+)/$', Run.as_view(), name='run-main'),
+		       url(r'file-upload/', FileUploadView.as_view(), name='file-upload'),
+		       #url(r'file-upload/(?P<filename>[-\w.]+)', FileUploadView.as_view(), name='file-upload'),
                        url(r'task/(?P<task_id>[-\w]+)/$', UserResult.as_view(), name='queue-task-result'),
                        url(r'usertasks/$', UserTasks.as_view(), name='queue-user-tasks'),
                        url(r'^$', Queue.as_view(), name="queue-main"),