diff --git a/.gitignore b/.gitignore index b424fd3..f1c6f94 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,10 @@ -.DS_Store -/dist -*.swp -*.pyc *.egg-info +*.pyc +*.swp +.cache .coverage -.tox +.DS_Store .eggs -.cache +.tox +/dist +/wheelhouse diff --git a/dj_elastictranscoder/__init__.py b/dj_elastictranscoder/__init__.py index fac5bf4..8a81504 100644 --- a/dj_elastictranscoder/__init__.py +++ b/dj_elastictranscoder/__init__.py @@ -1 +1 @@ -__version__ = '0.9.6' +__version__ = '1.0.4' diff --git a/dj_elastictranscoder/admin.py b/dj_elastictranscoder/admin.py index c7ad991..e5881bb 100644 --- a/dj_elastictranscoder/admin.py +++ b/dj_elastictranscoder/admin.py @@ -1,7 +1,10 @@ from django.contrib import admin + from .models import EncodeJob + class EncodeJobAdmin(admin.ModelAdmin): list_display = ('id', 'state', 'message') list_filters = ('state',) + admin.site.register(EncodeJob, EncodeJobAdmin) diff --git a/dj_elastictranscoder/models.py b/dj_elastictranscoder/models.py index 8422e07..e226543 100644 --- a/dj_elastictranscoder/models.py +++ b/dj_elastictranscoder/models.py @@ -1,7 +1,8 @@ from django.db import models from django.contrib.contenttypes.models import ContentType + import django -if django.get_version() >= '1.8': +if django.VERSION >= (1, 8): from django.contrib.contenttypes.fields import GenericForeignKey else: from django.contrib.contenttypes.generic import GenericForeignKey diff --git a/dj_elastictranscoder/signals.py b/dj_elastictranscoder/signals.py index f5c52c8..75cbb64 100644 --- a/dj_elastictranscoder/signals.py +++ b/dj_elastictranscoder/signals.py @@ -1,5 +1,6 @@ from django.dispatch import Signal -transcode_onprogress = Signal(providing_args=["job", "message"]) -transcode_onerror = Signal(providing_args=["job", "message"]) -transcode_oncomplete = Signal(providing_args=["job", "message"]) + +transcode_onprogress = Signal(providing_args=['job', 'job_response']) +transcode_oncomplete = Signal(providing_args=['job', 'job_response']) +transcode_onerror = Signal(providing_args=['job', 'job_response']) diff --git a/dj_elastictranscoder/transcoder.py b/dj_elastictranscoder/transcoder.py index 6d77f75..571271b 100644 --- a/dj_elastictranscoder/transcoder.py +++ b/dj_elastictranscoder/transcoder.py @@ -1,57 +1,173 @@ -from boto3.session import Session - -from django.conf import settings from django.contrib.contenttypes.models import ContentType from .models import EncodeJob +from .utils import get_setting_or_raise class Transcoder(object): - def __init__(self, pipeline_id, region=None, access_key_id=None, secret_access_key=None): - self.pipeline_id = pipeline_id + def start_job(self, obj, transcode_kwargs, message=''): + raise NotImplementedError() - if not region: - region = getattr(settings, 'AWS_REGION', None) - self.aws_region = region +class AWSTranscoder(Transcoder): + + def __init__(self, access_key_id=None, secret_access_key=None, pipeline_id=None, region=None): if not access_key_id: - access_key_id = getattr(settings, 'AWS_ACCESS_KEY_ID', None) - self.aws_access_key_id = access_key_id + access_key_id = get_setting_or_raise('AWS_ACCESS_KEY_ID') + self.access_key_id = access_key_id if not secret_access_key: - secret_access_key = getattr(settings, 'AWS_SECRET_ACCESS_KEY', None) - self.aws_secret_access_key = secret_access_key + secret_access_key = get_setting_or_raise('AWS_SECRET_ACCESS_KEY') + self.secret_access_key = secret_access_key - if self.aws_access_key_id is None: - assert False, 'Please provide AWS_ACCESS_KEY_ID' + if not pipeline_id: + pipeline_id = get_setting_or_raise('AWS_TRANSCODER_PIPELINE_ID') + self.pipeline_id = pipeline_id - if self.aws_secret_access_key is None: - assert False, 'Please provide AWS_SECRET_ACCESS_KEY' + if not region: + region = get_setting_or_raise('AWS_REGION') + self.region = region - if self.aws_region is None: - assert False, 'Please provide AWS_REGION' + from boto3.session import Session boto_session = Session( - aws_access_key_id=self.aws_access_key_id, - aws_secret_access_key=self.aws_secret_access_key, - region_name=self.aws_region, + aws_access_key_id=self.access_key_id, + aws_secret_access_key=self.secret_access_key, + region_name=self.region, ) self.client = boto_session.client('elastictranscoder') - def encode(self, input_name, outputs, **kwargs): - self.message = self.client.create_job( - PipelineId=self.pipeline_id, - Input=input_name, - Outputs=outputs, - **kwargs - ) + def start_job(self, obj, transcode_kwargs, message=''): + """ + https://boto3.readthedocs.io/en/latest/reference/services/elastictranscoder.html#ElasticTranscoder.Client.create_job + """ + + if 'PipelineId' not in transcode_kwargs: + transcode_kwargs['PipelineId'] = self.pipeline_id + + ret = self.client.create_job(**transcode_kwargs) - def create_job_for_object(self, obj): content_type = ContentType.objects.get_for_model(obj) + job = EncodeJob() + job.id = ret['Job']['Id'] + job.content_type = content_type + job.object_id = obj.pk + job.message = message + job.save() + + +class QiniuTranscoder(Transcoder): + + def __init__( + self, + access_key=None, + secret_key=None, + pipeline_id=None, + bucket_name=None, + notify_url=None, + ): + if not access_key: + access_key = get_setting_or_raise('QINIU_ACCESS_KEY') + self.access_key = access_key + + if not secret_key: + secret_key = get_setting_or_raise('QINIU_SECRET_KEY') + self.secret_key = secret_key + + if not pipeline_id: + pipeline_id = get_setting_or_raise('QINIU_TRANSCODE_PIPELINE_ID') + self.pipeline_id = pipeline_id + + if not bucket_name: + bucket_name = get_setting_or_raise('QINIU_TRANSCODE_BUCKET_NAME') + self.bucket_name = bucket_name + + if not notify_url: + notify_url = get_setting_or_raise('QINIU_TRANSCODE_NOTIFY_URL') + self.notify_url = notify_url + + from qiniu import Auth + + self.client = Auth(self.access_key, self.secret_key) + + def start_job(self, obj, transcode_kwargs, message=''): + """ + https://developer.qiniu.com/dora/manual/1248/audio-and-video-transcoding-avthumb + """ + + from qiniu import PersistentFop + if 'force' not in transcode_kwargs: + transcode_kwargs['force'] = 1 + + pfop = PersistentFop(self.client, self.bucket_name, self.pipeline_id, self.notify_url) + ret, info = pfop.execute(**transcode_kwargs) + + content_type = ContentType.objects.get_for_model(obj) + job = EncodeJob() + job.id = ret['persistentId'] + job.content_type = content_type + job.object_id = obj.pk + job.message = message + job.save() + + +class AliyunTranscoder(Transcoder): + + def __init__( + self, + access_key_id=None, + access_key_secret=None, + pipeline_id=None, + region=None, + notify_url=None + ): + if not access_key_id: + access_key_id = get_setting_or_raise('ALIYUN_TRANSCODE_ACCESS_KEY_ID') + self.access_key_id = access_key_id + + if not access_key_secret: + access_key_secret = get_setting_or_raise('ALIYUN_TRANSCODE_ACCESS_KEY_SECRET') + self.access_key_secret = access_key_secret + + if not pipeline_id: + pipeline_id = get_setting_or_raise('ALIYUN_TRANSCODE_PIPELINE_ID') + self.pipeline_id = pipeline_id + + if not region: + region = get_setting_or_raise('ALIYUN_TRANSCODE_REGION') + self.region = region + + if not notify_url: + notify_url = get_setting_or_raise('ALIYUN_TRANSCODE_NOTIFY_URL') + self.notify_url = notify_url + + from aliyunsdkcore import client + + self.client = client.AcsClient(self.access_key_id, self.access_key_secret, self.region) + + def start_job(self, obj, transcode_kwargs, message=''): + """ + https://help.aliyun.com/document_detail/57347.html?spm=5176.doc56767.6.724.AJ8z3E + """ + + import json + from aliyunsdkmts.request.v20140618 import SubmitJobsRequest + + request = SubmitJobsRequest.SubmitJobsRequest() + request.set_accept_format('json') + request.set_Input(json.dumps(transcode_kwargs.get('input_file'))) + request.set_OutputBucket(transcode_kwargs.get('bucket')) + request.set_OutputLocation(transcode_kwargs.get('oss_location')) + request.set_Outputs(json.dumps(transcode_kwargs.get('outputs'))) + request.set_PipelineId(self.pipeline_id) + response = json.loads(self.client.do_action_with_exception(request).decode('utf-8')) + + content_type = ContentType.objects.get_for_model(obj) job = EncodeJob() - job.id = self.message['Job']['Id'] + job.id = response['JobResultList']['JobResult'][0]['Job']['JobId'] job.content_type = content_type job.object_id = obj.pk + job.message = message job.save() diff --git a/dj_elastictranscoder/urls.py b/dj_elastictranscoder/urls.py index 3b6d4a3..7ae4e8c 100644 --- a/dj_elastictranscoder/urls.py +++ b/dj_elastictranscoder/urls.py @@ -1,8 +1,27 @@ -try: - from django.conf.urls import url, patterns -except ImportError: - from django.conf.urls.defaults import url, patterns # Support for Django < 1.4 - -urlpatterns = patterns('dj_elastictranscoder.views', - url(http://github.com/r'%5Eendpoint/'),%20'endpoint'), -) +import django + + +if django.VERSION >= (1, 9): + from django.conf.urls import url + from dj_elastictranscoder import views + + urlpatterns = [ + url(http://github.com/r'%5Eendpoint/),%20views.aws_endpoint), + url(http://github.com/r'%5Eaws_endpoint/'),%20views.aws_endpoint,%20name='aws_endpoint'), + url(http://github.com/r'%5Eqiniu_endpoint/'),%20views.qiniu_endpoint,%20name='qiniu_endpoint'), + url(http://github.com/r'%5Ealiyun_endpoint',%20views.aliyun_endpoint,%20name='aliyun_endpoint'), + ] + +else: + try: + from django.conf.urls import url, patterns + except ImportError: + from django.conf.urls.defaults import url, patterns # Support for Django < 1.4 + + urlpatterns = patterns( + 'dj_elastictranscoder.views', + url(http://github.com/r'%5Eendpoint/'),%20'aws_endpoint'), + url(http://github.com/r'%5Eaws_endpoint/'),%20'aws_endpoint',%20name='aws_endpoint'), + url(http://github.com/r'%5Eqiniu_endpoint/'),%20'qiniu_endpoint',%20name='qiniu_endpoint'), + url(http://github.com/r'%5Ealiyun_endpoint',%20'aliyun_endpoint',%20name='aliyun_endpoint'), + ) diff --git a/dj_elastictranscoder/utils.py b/dj_elastictranscoder/utils.py new file mode 100644 index 0000000..c31e5e9 --- /dev/null +++ b/dj_elastictranscoder/utils.py @@ -0,0 +1,10 @@ +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured + + +def get_setting_or_raise(setting_name): + try: + value = getattr(settings, setting_name) + except AttributeError: + raise ImproperlyConfigured('Please provide {0}'.format(setting_name)) + return value diff --git a/dj_elastictranscoder/views.py b/dj_elastictranscoder/views.py index 9ff0f51..a292f08 100644 --- a/dj_elastictranscoder/views.py +++ b/dj_elastictranscoder/views.py @@ -1,8 +1,9 @@ import json -from django.http import HttpResponse, HttpResponseBadRequest -from django.views.decorators.csrf import csrf_exempt from django.core.mail import mail_admins +from django.http import Http404, HttpResponse, HttpResponseBadRequest +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_http_methods from .models import EncodeJob from .signals import ( @@ -11,18 +12,19 @@ transcode_oncomplete ) + @csrf_exempt -def endpoint(request): +def aws_endpoint(request): """ Receive SNS notification """ try: - data = json.loads(request.read().decode('utf-8')) + webhook = request.read().decode('utf-8') + data = json.loads(webhook) except ValueError: return HttpResponseBadRequest('Invalid JSON') - # handle SNS subscription if data['Type'] == 'SubscriptionConfirmation': subscribe_url = data['SubscribeURL'] @@ -34,34 +36,108 @@ def endpoint(request): mail_admins('Please confirm SNS subscription', subscribe_body) return HttpResponse('OK') - - # + # handle job response + message = json.loads(data['Message']) + state = message['state'] + + job = EncodeJob.objects.get(pk=message['jobId']) + + # https://docs.aws.amazon.com/elastictranscoder/latest/developerguide/notifications.html + if state == 'PROGRESSING': + job.message = webhook + job.state = 1 + job.save() + transcode_onprogress.send(sender=None, job=job, job_response=data) + elif state == 'COMPLETED': + job.message = webhook + job.state = 4 + job.save() + transcode_oncomplete.send(sender=None, job=job, job_response=data) + elif state == 'ERROR': + job.message = webhook + job.state = 2 + job.save() + transcode_onerror.send(sender=None, job=job, job_response=data) + else: + raise RuntimeError('Invalid state') + + return HttpResponse('Done') + + +@csrf_exempt +@require_http_methods(['POST', ]) +def qiniu_endpoint(request): + """ + Receive Qiniu notification + """ + try: - message = json.loads(data['Message']) + webhook = request.read().decode('utf-8') + data = json.loads(webhook) except ValueError: - assert False, data['Message'] + return HttpResponseBadRequest('Invalid JSON') + + code = data['code'] + job_id = data['id'] - # - if message['state'] == 'PROGRESSING': - job = EncodeJob.objects.get(pk=message['jobId']) - job.message = 'Progress' + job = EncodeJob.objects.get(pk=job_id) + + # https://developer.qiniu.com/dora/manual/1294/persistent-processing-status-query-prefop + if code in (1, 2): # Progressing + job.message = webhook job.state = 1 job.save() - - transcode_onprogress.send(sender=None, job=job, message=message) - elif message['state'] == 'COMPLETED': - job = EncodeJob.objects.get(pk=message['jobId']) - job.message = 'Success' + transcode_onprogress.send(sender=None, job=job, job_response=data) + elif code == 0: # Complete + job.message = webhook job.state = 4 job.save() - - transcode_oncomplete.send(sender=None, job=job, message=message) - elif message['state'] == 'ERROR': - job = EncodeJob.objects.get(pk=message['jobId']) - job.message = message['messageDetails'] + transcode_oncomplete.send(sender=None, job=job, job_response=data) + elif code == 3 or code == 4: # Error + job.message = webhook job.state = 2 job.save() - - transcode_onerror.send(sender=None, job=job, message=message) + transcode_onerror.send(sender=None, job=job, job_response=data) + else: + raise RuntimeError('Invalid code') return HttpResponse('Done') + + +@csrf_exempt +@require_http_methods(['POST', ]) +def aliyun_endpoint(request): + """ + Receive Aliyun notification + """ + + try: + webhook = request.read().decode('utf-8') + data = json.loads(webhook) + except ValueError: + return HttpResponseBadRequest('Invalid JSON') + + message = json.loads(data['Message']) + if message['Type'] == 'Transcode': + state = message['state'] + job_id = message['jobId'] + + try: + job = EncodeJob.objects.get(pk=job_id) + except EncodeJob.DoesNotExist: + raise Http404 + + # https://help.aliyun.com/document_detail/57347.html?spm=5176.doc29208.6.724.4zQQQ4 + if state == 'Success': # Complate + job.message = webhook + job.state = 4 + job.save() + transcode_oncomplete.send(sender=None, job=job, job_response=job_id) + elif state == 'Fail': # Error + job.message = webhook + job.state = 2 + job.save() + transcode_onerror.send(sender=None, job=job, job_response=data) + else: + raise RuntimeError('Invalid code') + return HttpResponse('Done') diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..2a9acf1 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/setup.py b/setup.py index 13ecc92..6268e86 100644 --- a/setup.py +++ b/setup.py @@ -1,23 +1,44 @@ +import os +import sys + from setuptools import setup, find_packages -from dj_elastictranscoder import __version__ +def get_version(): + code = None + path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'dj_elastictranscoder', + '__init__.py', + ) + with open(path) as f: + for line in f: + if line.startswith('__version__'): + code = line[len('__version__ = '):] + break + return eval(code) + + +if sys.argv[-1] == 'wheel': + os.system('pip wheel --wheel-dir=wheelhouse .') + sys.exit() + setup( name='django-elastic-transcoder', - version=__version__, + version=get_version(), description="Django with AWS elastic transcoder", - long_description=open("README.rst").read(), + long_description=open('README.rst').read(), author='tzangms', author_email='tzangms@streetvoice.com', url='http://github.com/StreetVoice/django-elastic-transcoder', license='MIT', - packages=find_packages(), + packages=find_packages(exclude=('testsapp', )), include_package_data=True, zip_safe=False, install_requires=[ - "django >= 1.3, < 1.9", "boto3 >= 1.1", - "South >= 0.8", + "django >= 1.3, < 2.0", + "qiniu >= 7.0.8", ], classifiers=[ "Intended Audience :: Developers", @@ -31,5 +52,5 @@ "Environment :: Web Environment", "Framework :: Django", ], - keywords='django,aws,elastic,transcoder', + keywords='django,aws,elastic,transcoder,qiniu,audio,aliyun', )