jm2l/jm2l/upload.py

# -*- coding: utf8 -*-
from pyramid.view import view_config, view_defaults
from pyramid.response import Response
from pyramid.exceptions import NotFound
from pyramid.httpexceptions import HTTPNotFound
from pyramid.request import Request
from PIL import Image
import re, os, shutil
from os import path
import mimetypes
import magic
import subprocess
import cStringIO as StringIO
# Database access imports
from .models import User, Place, Tiers, Event, SallePhy

CurrentYear = 2015
MIN_FILE_SIZE = 1 # bytes
MAX_FILE_SIZE = 500000000 # bytes
IMAGE_TYPES = re.compile('image/(gif|p?jpeg|(x-)?png)')
ACCEPTED_MIMES = ['application/pdf',
    'application/vnd.oasis.opendocument.text',
    'application/vnd.oasis.opendocument.text-template',
    'application/vnd.oasis.opendocument.graphics',
    'application/vnd.oasis.opendocument.graphics-template',
    'application/vnd.oasis.opendocument.presentation',
    'application/vnd.oasis.opendocument.presentation-template',
    'application/vnd.oasis.opendocument.spreadsheet',
    'application/vnd.oasis.opendocument.spreadsheet-template',
    'image/svg+xml'
]

        

ACCEPT_FILE_TYPES = IMAGE_TYPES
THUMBNAIL_SIZE = 80
EXPIRATION_TIME = 300 # seconds
IMAGEPATH = [ 'images' ]
DOCPATH = [ 'document' ]
THUMBNAILPATH = [ 'images', 'thumbnails' ]
# change the following to POST if DELETE isn't supported by the webserver
DELETEMETHOD="DELETE"

mimetypes.init()

class MediaPath(): 
    
    def get_list(self, media_table, linked_id, MediaType=None):
        filelist = list()
        curpath = self.get_mediapath(media_table, linked_id, None)
        if not os.path.isdir(curpath):
            return list()
        for f in os.listdir(curpath):
            if os.path.isdir(os.path.join(curpath,f)):
                continue
            if f.endswith('.type'):
                continue
            if f:
                filename, ext = os.path.splitext( f )
                tmpurl = '/image/%s/%d/%s' % (media_table, linked_id, f.replace(" ","%20"))
                if MediaType is None:
                    filelist.append(tmpurl)
                elif MediaType=='Image' and ext.lower() in ['.gif','.jpg','.png','.svg','.jpeg']:
                    filelist.append(tmpurl)
                elif MediaType=='Other' and ext.lower() not in ['.gif','.jpg','.png','.svg','.jpeg']:
                    filelist.append(tmpurl)
        return filelist
        
    def get_thumb(self, media_table, linked_id, MediaType=None):
        filelist = list()
        curpath = self.get_mediapath(media_table, linked_id, None)
        curpath = os.path.join( curpath, 'thumbnails')
        if not os.path.isdir(curpath):
            return list()
        for f in os.listdir(curpath):
            filename, ext = os.path.splitext( f )
            if os.path.isdir(os.path.join(curpath,f)):
                continue
            if f.endswith('.type'):
                continue
            if f:
                tmpurl = '/image/%s/%d/thumbnails/%s' % (media_table, linked_id, f.replace(" ","%20"))
                if MediaType is None:
                    filelist.append(tmpurl)
                elif MediaType=='Image' and len( os.path.splitext(filename)[1] )==0:
                    filelist.append(tmpurl)
                elif MediaType=='Other' and len( os.path.splitext(filename)[1] ):
                    filelist.append(tmpurl)
        return filelist          
    
    def get_mediapath(self, media_table, linked_id, name):
        
        linked_id = str(linked_id)
        if media_table in ['tiers', 'place', 'salle']:
            # Retrieve Slug            
            if media_table=='tiers':
                slug = Tiers.by_id(linked_id).slug
            if media_table=='place':
                slug = Place.by_id(linked_id).slug
            if media_table=='salle':
                slug = SallePhy.by_id(linked_id).slug
            p = IMAGEPATH + [ media_table ] + [ slug ]
        elif media_table=='presse':
            # Use Year in linked_id
            p = IMAGEPATH + [ media_table ] + [ linked_id ]
        elif media_table=='tasks':
            # Use Current Year
            p = IMAGEPATH + [ str(CurrentYear), media_table ] + [ linked_id ]
        elif media_table=='poles':
            # Use Current Year
            p = IMAGEPATH + [ str(CurrentYear), media_table ] + [ linked_id ]
        elif media_table in ['RIB', 'Justif']:
            slug = User.by_id(linked_id).slug
            p = IMAGEPATH + ['users'] + [ slug ] + [ self.media_table ]
        elif media_table=='users':
            user = User.by_id(linked_id)
            if not user:
                raise HTTPNotFound()
            else:
                slug = user.slug
            p = IMAGEPATH + ['users'] + [ slug ]
        elif media_table=='event':
            ev = Event.by_id(linked_id)
            slug = ev.slug
            year = ev.for_year
            p = IMAGEPATH + ['event'] + [ str(year) ] + [ slug ]

        if name:
            p += [ name ]
        TargetPath = os.path.join('jm2l/upload', *p)
        if not os.path.isdir(os.path.dirname(TargetPath)):
            os.makedirs(os.path.dirname(TargetPath))
        return os.path.join('jm2l/upload', *p) 

    def ExtMimeIcon(self, mime):
        if mime=='application/pdf':
            return "/img/PDF.png"

@view_defaults(route_name='media_upload')
class MediaUpload(MediaPath):

    def __init__(self, request):
        self.request = request
        self.media_table = self.request.matchdict.get('media_table')
        self.linked_id = self.request.matchdict.get('uid')
        if not self.linked_id.isdigit():
            raise HTTPBadRequest('Wrong Parameter')
        request.response.headers['Access-Control-Allow-Origin'] = '*'
        request.response.headers['Access-Control-Allow-Methods'] = 'OPTIONS, HEAD, GET, POST, PUT, DELETE'

    def mediapath(self, name):
        return self.get_mediapath(self.media_table, self.linked_id, name)

    def validate(self, result, filecontent):
        # let's say we don't trust the uploader
        RealMime = magic.from_buffer( filecontent.read(1024), mime=True)
        filecontent.seek(0)
        if RealMime!=result['type']:
            result['error'] = 'l\'extension du fichier ne correspond pas à son contenu - '
            result['error'] += "( %s vs %s )" % (RealMime, result['type'])
            return False
        # Accept images and mime types listed
        if not RealMime in ACCEPTED_MIMES:
            if not (IMAGE_TYPES.match(RealMime)):
                result['error'] = 'Ce type fichier n\'est malheureusement pas supporté. '
                result['error'] += 'Les fichiers acceptées sont les images et pdf.'
                return False
        if result['size'] < MIN_FILE_SIZE:
            result['error'] = 'le fichier est trop petit'
        elif result['size'] > MAX_FILE_SIZE:
            result['error'] = 'le fichier est trop voluminueux'
        #elif not ACCEPT_FILE_TYPES.match(file['type']):
        #    file['error'] = u'les type de fichiers acceptés sont png, jpg et gif'
        else:
            return True
        return False

    def get_file_size(self, file):
        file.seek(0, 2) # Seek to the end of the file
        size = file.tell() # Get the position of EOF
        file.seek(0) # Reset the file position to the beginning
        return size

    def thumbnailurl(self,name):
        return self.request.route_url('media_view',name='thumbnails',
                media_table=self.media_table,
                uid=self.linked_id) + '/' + name

    def thumbnailpath(self,name):
        origin = self.mediapath(name)
        TargetPath = os.path.join( os.path.dirname(origin), 'thumbnails', name)
        if not os.path.isdir(os.path.dirname(TargetPath)):
            os.makedirs(os.path.dirname(TargetPath))        
        return TargetPath

    def createthumbnail(self, filename):
        image = Image.open( self.mediapath(filename) )
        image.thumbnail((THUMBNAIL_SIZE, THUMBNAIL_SIZE), Image.ANTIALIAS)
        timage = Image.new('RGBA', (THUMBNAIL_SIZE, THUMBNAIL_SIZE), (255, 255, 255, 0))
        timage.paste(
            image,
            ((THUMBNAIL_SIZE - image.size[0]) / 2, (THUMBNAIL_SIZE - image.size[1]) / 2))
        TargetFileName = self.thumbnailpath(filename)
        timage.save( TargetFileName )
        return self.thumbnailurl( os.path.basename(TargetFileName) )

    def pdfthumbnail(self, filename):
        TargetFileName = self.thumbnailpath(filename)
        Command = ["convert","./%s[0]" % self.mediapath(filename),"./%s_.jpg" % TargetFileName]
        Result = subprocess.call(Command)
        if Result==0:
            image = Image.open( TargetFileName+"_.jpg" )
            pdf_indicator = Image.open( "jm2l/static/img/PDF_Thumb_Stamp.png" )
            image.thumbnail((THUMBNAIL_SIZE, THUMBNAIL_SIZE), Image.ANTIALIAS)
            timage = Image.new('RGBA', (THUMBNAIL_SIZE, THUMBNAIL_SIZE), (255, 255, 255, 0))
            # Add thumbnail
            timage.paste(
                image, 
                ((THUMBNAIL_SIZE - image.size[0]) / 2, (THUMBNAIL_SIZE - image.size[1]) / 2))
            # Stamp with PDF file type
            timage.paste(
                pdf_indicator,
                (timage.size[0]-30, timage.size[1]-30),
                pdf_indicator,
                )
            timage.convert('RGB').save( TargetFileName+".jpg", 'JPEG')
            os.unlink(TargetFileName+"_.jpg")
            return self.thumbnailurl( os.path.basename(TargetFileName+".jpg") )
        return self.ExtMimeIcon('application/pdf')

    def svgthumbnail(self, filename):
        TargetFileName = self.thumbnailpath(filename)
        Command = ["convert","./%s[0]" % self.mediapath(filename),"./%s_.jpg" % TargetFileName]
        Result = subprocess.call(Command)
        if Result==0:
            image = Image.open( TargetFileName+"_.jpg" )
            pdf_indicator = Image.open( "jm2l/static/img/svg-icon.png" )
            image.thumbnail((THUMBNAIL_SIZE, THUMBNAIL_SIZE), Image.ANTIALIAS)
            timage = Image.new('RGBA', (THUMBNAIL_SIZE, THUMBNAIL_SIZE), (255, 255, 255, 0))
            # Add thumbnail
            timage.paste(
                image, 
                ((THUMBNAIL_SIZE - image.size[0]) / 2, (THUMBNAIL_SIZE - image.size[1]) / 2))
            # Stamp with PDF file type
            timage.paste(
                pdf_indicator,
                (timage.size[0]-30, timage.size[1]-30),
                pdf_indicator,
                )
            timage.convert('RGB').save( TargetFileName+".jpg", 'JPEG')
            os.unlink(TargetFileName+"_.jpg")
            return self.thumbnailurl( os.path.basename(TargetFileName+".jpg") )
        return self.ExtMimeIcon('image/svg+xml')

    def docthumbnail(self, filename):
        TargetFileName = self.thumbnailpath(filename)
        # unoconv need a libre office server to be up
        Command = ["unoconv", "-f", "pdf", "-e", "PageRange=1", "--output=%s" % TargetFileName, \
            "%s[0]" % self.mediapath(filename) ]
        # let's take the thumbnail generated inside the document
        Command = ["unzip", "-p", self.mediapath(filename), "Thumbnails/thumbnail.png"]
        ThumbBytes = subprocess.check_output(Command)
        image = Image.open( StringIO.StringIO(ThumbBytes) )
        image.thumbnail((THUMBNAIL_SIZE, THUMBNAIL_SIZE), Image.ANTIALIAS)
        # Use the correct stamp
        f, ext = os.path.splitext( filename )
        istamp = [  ('Writer','odt'),
                    ('Impress','odp'),
                    ('Calc','ods'),
                    ('Draw','odg')]
        stampfilename = filter(lambda (x,y): ext.endswith(y), istamp)
        stamp = Image.open( "jm2l/static/img/%s-icon.png" % stampfilename[0][0])
        timage = Image.new('RGBA', (THUMBNAIL_SIZE, THUMBNAIL_SIZE), (255, 255, 255, 0))
        # Add thumbnail
        timage.paste(
            image, 
            ((THUMBNAIL_SIZE - image.size[0]) / 2, (THUMBNAIL_SIZE - image.size[1]) / 2))
        # Stamp with PDF file type
        timage.paste(
            stamp,
            (timage.size[0]-30, timage.size[1]-30),
            stamp,
            )
        timage.convert('RGB').save( TargetFileName+".jpg", 'JPEG')
        return self.thumbnailurl( os.path.basename(TargetFileName+".jpg") )

    def fileinfo(self,name):
        filename = self.mediapath(name) 
        f, ext = os.path.splitext(name)
        if ext!='.type' and os.path.isfile(filename):
            info = {}
            info['name'] = name
            info['size'] = os.path.getsize(filename)
            info['url'] = self.request.route_url('media_view',
                        name=name,
                        media_table=self.media_table,
                        uid=self.linked_id)
            mime = mimetypes.types_map.get(ext)
            if (IMAGE_TYPES.match(mime)):
                info['thumbnailUrl'] = self.thumbnailurl(name)
            elif mime in ACCEPTED_MIMES:
                thumb = self.thumbnailpath("%s%s" % (f, ext))
                if os.path.exists( thumb +'.jpg' ):
                    info['thumbnailUrl'] = self.thumbnailurl(name)+'.jpg'
                else:
                    info['thumbnailUrl'] = self.ExtMimeIcon(mime)
            else:
                info['thumbnailUrl'] = self.ExtMimeIcon(mime)
            info['deleteType'] = DELETEMETHOD
            info['deleteUrl'] = self.request.route_url('media_upload',
                        sep='',
                        name='',                        
                        media_table=self.media_table,
                        uid=self.linked_id) + '/' + name
            if DELETEMETHOD != 'DELETE':
                info['deleteUrl'] += '&_method=DELETE'
            return info
        else:
            return None

    @view_config(request_method='OPTIONS')
    def options(self):
        return Response(body='')

    @view_config(request_method='HEAD')
    def options(self):
        return Response(body='')

    @view_config(request_method='GET', renderer="json")
    def get(self):
        p = self.request.matchdict.get('name')
        if p:
            return self.fileinfo(p)
        else:
            filelist = []
            content = self.mediapath('')
            if content and path.exists(content):
                for f in os.listdir(content):
                    n = self.fileinfo(f)
                    if n:
                        filelist.append(n)
            return { "files":filelist }

    @view_config(request_method='DELETE', xhr=True, accept="application/json", renderer='json')
    def delete(self):
        import json
        filename = self.request.matchdict.get('name')
        try:
            os.remove(self.mediapath(filename) + '.type')
        except IOError:
            pass
        except OSError:
            pass                
        try:
            os.remove(self.thumbnailpath(filename))
        except IOError:
            pass
        except OSError:       
            pass
        try:
            os.remove(self.thumbnailpath(filename+".jpg"))
        except IOError:
            pass
        except OSError:         
            pass            
        try:
            os.remove(self.mediapath(filename))
        except IOError:
            return False
        return True
    
    @view_config(request_method='POST', xhr=True, accept="application/json", renderer='json')
    def post(self):
        if self.request.matchdict.get('_method') == "DELETE":
            return self.delete()
        results = []
        for name, fieldStorage in self.request.POST.items():
            if isinstance(fieldStorage,unicode):
                continue
            result = {}
            result['name'] = os.path.basename(fieldStorage.filename)
            result['type'] = fieldStorage.type
            result['size'] = self.get_file_size(fieldStorage.file)
            if self.validate(result, fieldStorage.file):
                with open( self.mediapath(result['name'] + '.type'), 'w') as f:
                    f.write(result['type'])
                with open( self.mediapath(result['name']), 'wb') as f:
                    shutil.copyfileobj( fieldStorage.file , f)
                if re.match(IMAGE_TYPES, result['type']):
                    result['thumbnailUrl'] = self.createthumbnail(result['name'])
                elif result['type']=='application/pdf':
                    result['thumbnailUrl'] = self.pdfthumbnail(result['name'])
                elif result['type']=='image/svg+xml':
                    result['thumbnailUrl'] = self.svgthumbnail(result['name'])
                elif result['type'].startswith('application/vnd'):
                    result['thumbnailUrl'] = self.docthumbnail(result['name'])
                else:
                    result['thumbnailUrl'] = self.ExtMimeIcon(result['type'])
                result['deleteType'] = DELETEMETHOD
                result['deleteUrl'] = self.request.route_url('media_upload',
                            sep='',
                            name='',
                            media_table=self.media_table,
                            uid=self.linked_id) + '/' + result['name']
                result['url'] = self.request.route_url('media_view',
                            media_table=self.media_table,
                            uid=self.linked_id,
                            name=result['name'])
                if DELETEMETHOD != 'DELETE':
                    result['deleteUrl'] += '&_method=DELETE'
            results.append(result)
        return {"files":results}

@view_defaults(route_name='media_view')
class MediaView(MediaPath):

    def __init__(self,request):
        self.request = request
        self.media_table = self.request.matchdict.get('media_table')
        self.linked_id = self.request.matchdict.get('uid')

    def mediapath(self,name):
        return self.get_mediapath(self.media_table, self.linked_id, name)
        
    @view_config(request_method='GET', http_cache = (EXPIRATION_TIME, {'public':True}))
    def get(self):
        name = self.request.matchdict.get('name')
        try:
            with open( self.mediapath( os.path.basename(name) ) + '.type', 'r', 16) as f:
                test = f.read()
                self.request.response.content_type = test
        except IOError:
            pass
        try:
            self.request.response.body_file = open( self.mediapath(name), 'rb', 10000)
        except IOError:
            raise NotFound
        return self.request.response
    ##return Response(app_iter=ImgHandle, content_type = 'image/png')