* Update codebase to support python 3.8

certsocietegenerale · Feb 9, 2021 · 06abcb1 · 06abcb1
1 parent 3f46f8f
commit 06abcb1
Show file tree

Hide file tree

Showing 81 changed files with 376 additions and 241 deletions.
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,7 +1,8 @@
-FROM python:2.7-alpine AS builder
+FROM python:3.8-alpine AS builder
 WORKDIR /app
 COPY . /app
-RUN apk add --update python-dev \
+RUN apk add --update git \
+    python3-dev \
     libxml2-dev \
     libxslt-dev \
     build-base \
@@ -16,7 +17,7 @@ RUN apk add --update python-dev \
     mv fir/config/installed_apps.txt.sample fir/config/installed_apps.txt && \
     deactivate
 
-FROM python:2.7-alpine AS fir
+FROM python:3.8-alpine AS fir
 COPY --from=builder /app /app
 RUN apk add libxml2 libxslt mariadb-connector-c && \
     rm -rf /var/cache/apk/* && \

diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -67,7 +67,7 @@ services:
       - fir_db
       - fir_redis
     env_file:
-      - fir.env  
+      - fir.env
     networks:
       backend.fir:
 

diff --git a/docker/fir.env b/docker/fir.env
@@ -4,7 +4,7 @@ HOME=/app/FIR
 DJANGO_SETTINGS_MODULE=fir.config.composeprod
 
 ALLOWED_HOSTS=localhost,127.0.0.1
-DEBUG=true
+DEBUG=false
 
 # Secret key
 SECRET_KEY="not so secret"

diff --git a/fir/config/base.py b/fir/config/base.py
@@ -71,7 +71,7 @@
 #    'django.contrib.staticfiles.finders.DefaultStorageFinder',
 )
 
-MIDDLEWARE_CLASSES = (
+MIDDLEWARE = (
     'django.middleware.common.CommonMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',

diff --git a/fir/config/composeprod.py b/fir/config/composeprod.py
@@ -96,5 +96,11 @@
     },
 }
 
+# Rest framework default pagination
+REST_FRAMEWORK = {
+    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
+    'PAGE_SIZE': 100
+}
+
 # External URL of your FIR application (used in fir_notification to render full URIs in templates)
 #EXTERNAL_URL = 'http://fir.example.com'
diff --git a/fir/urls.py b/fir/urls.py
@@ -5,17 +5,18 @@
 from fir.config.base import INSTALLED_APPS, TF_INSTALLED
 from incidents import views
 
+
 # urls for core FIR components
 urlpatterns = [
     url(r'^logout/', views.user_logout, name='logout'),
-    url(r'^incidents/', include('incidents.urls', namespace='incidents')),
+    url(r'^incidents/', include(('incidents.urls', 'incidents'), namespace='incidents')),
     url(r'^search/$', views.search, name='search'),
-    url(r'^events/', include('incidents.custom_urls.events', namespace='events')),
-    url(r'^stats/', include('incidents.custom_urls.stats', namespace='stats')),
-    url(r'^ajax/', include('incidents.custom_urls.ajax', namespace='ajax')),
-    url(r'^user/', include('incidents.custom_urls.user', namespace='user')),
-    url(r'^dashboard/', include('incidents.custom_urls.dashboard', namespace='dashboard')),
-    url(r'^admin/', include(admin.site.urls)),
+    url(r'^events/', include(('incidents.custom_urls.events', 'url_events'), namespace='events')),
+    url(r'^stats/', include(('incidents.custom_urls.stats', 'stats'), namespace='stats')),
+    url(r'^ajax/', include(('incidents.custom_urls.ajax', 'ajax'), namespace='ajax')),
+    url(r'^user/', include(('incidents.custom_urls.user', 'user'), namespace='user')),
+    url(r'^dashboard/', include(('incidents.custom_urls.dashboard', 'dashboard'), namespace='dashboard')),
+    url(r'^admin/', admin.site.urls),
     url(r'^$', views.dashboard_main),
 ]
 
@@ -39,4 +40,4 @@
         app_name = app[4:]
         app_urls = '{}.urls'.format(app)
         if find_loader(app_urls):
-            urlpatterns.append(url('^{}/'.format(app_name), include(app_urls, namespace=app_name)))
+            urlpatterns.append(url('^{}/'.format(app_name), include((app_urls, app), namespace=app_name)))
diff --git a/fir_abuse/models.py b/fir_abuse/models.py
@@ -9,9 +9,9 @@ class AbuseTemplate(models.Model):
     type = models.CharField(max_length=100, blank=True)
     body = models.TextField()
     subject = models.TextField()
-    incident_category = models.ForeignKey(IncidentCategory, blank=True, null=True)
+    incident_category = models.ForeignKey(IncidentCategory, on_delete=models.CASCADE, blank=True, null=True)
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
 
@@ -20,10 +20,10 @@ class AbuseContact(models.Model):
     to = models.CharField(max_length=100)
     cc = models.CharField(max_length=100, blank=True)
     bcc = models.CharField(max_length=100, blank=True)
-    incident_category = models.ForeignKey(IncidentCategory, blank=True, null=True)
+    incident_category = models.ForeignKey(IncidentCategory, on_delete=models.CASCADE, blank=True, null=True)
     type = models.CharField(max_length=100, blank=True)
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
 

diff --git a/fir_abuse/templates/fir_abuse/plugins/details_static.html b/fir_abuse/templates/fir_abuse/plugins/details_static.html
@@ -1,3 +1,3 @@
-{% load staticfiles %}
+{% load static %}
 <script src="{% static "fir_abuse/js/abuse.js" %}"></script>
 <link href="{% static "fir_abuse/css/abuse.css" %}" rel="stylesheet" />
diff --git a/fir_abuse/urls.py b/fir_abuse/urls.py
@@ -2,6 +2,8 @@
 
 from fir_abuse import views
 
+app_name='fir_abuse'
+
 urlpatterns = [
     url(r'^(?P<incident_id>\d+)/get_template/(?P<artifact_id>\d+)/$', views.get_template, name='get_template'),
     url(r'^emailform/$', views.emailform, name='emailform'),

diff --git a/fir_abuse/views.py b/fir_abuse/views.py
@@ -39,7 +39,7 @@ def send_email(request):
 
             return HttpResponse(dumps({'status': 'ok'}), content_type="application/json")
 
-        except Exception, e:
+        except Exception as e:
             return HttpResponse(dumps({'status': 'ko', 'error': str(e)}), content_type="application/json")
 
     return HttpResponseBadRequest(dumps({'status': 'ko'}), content_type="application/json")

diff --git a/fir_alerting/migrations/0003_auto_20210209_0717.py b/fir_alerting/migrations/0003_auto_20210209_0717.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.1.6 on 2021-02-09 07:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fir_alerting', '0002_add_helptext_to_body'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='categorytemplate',
+            name='body',
+            field=models.TextField(help_text='This is a Markdown field. You can use django templating language.'),
+        ),
+    ]
diff --git a/fir_alerting/models.py b/fir_alerting/models.py
@@ -11,9 +11,9 @@ class RecipientTemplate(models.Model):
     recipient_to = models.TextField()
     recipient_cc = models.TextField()
     recipient_bcc = models.TextField(null=True, blank=True)
-    business_line = models.ForeignKey(BusinessLine, null=True, blank=True)
+    business_line = models.ForeignKey(BusinessLine, on_delete=models.CASCADE, null=True, blank=True)
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
     class Meta:
@@ -25,9 +25,9 @@ class CategoryTemplate(models.Model):
     type = models.CharField(max_length=100)
     body = models.TextField(help_text="This is a Markdown field. You can use django templating language.")
     subject = models.TextField()
-    incident_category = models.ForeignKey(IncidentCategory)
+    incident_category = models.ForeignKey(IncidentCategory, on_delete=models.CASCADE)
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
     class Meta:

diff --git a/fir_alerting/templates/fir_alerting/plugins/details_static.html b/fir_alerting/templates/fir_alerting/plugins/details_static.html
@@ -1,3 +1,3 @@
-{% load staticfiles %}
+{% load static %}
 <script src="{% static "fir_alerting/js/alerting.js" %}"></script>
 <link href="{% static "fir_alerting/css/alerting.css" %}" rel="stylesheet" />
diff --git a/fir_alerting/urls.py b/fir_alerting/urls.py
@@ -2,6 +2,8 @@
 
 from fir_alerting import views
 
+app_name='fir_alerting'
+
 urlpatterns = [
     url(r'^(?P<incident_id>\d+)/get_template/(?P<template_type>[\w-]+)/$', views.get_template, name='get_template'),
     url(r'^(?P<incident_id>\d+)/get_template/(?P<template_type>[\w-]+)/(?P<bl>[\d]+)/$', views.get_template, name='get_template'),

diff --git a/fir_alerting/views.py b/fir_alerting/views.py
@@ -41,7 +41,7 @@ def get_template(request, incident_id, template_type, bl=None, authorization_tar
 
     try:
         cat_template = CategoryTemplate.objects.get(incident_category=i.category, type=template_type)
-    except Exception, e:
+    except Exception as e:
         cat_template = None
 
     rec_template = None
@@ -119,7 +119,7 @@ def send_email(request):
 
             return HttpResponse(dumps({'status': 'ok'}), content_type="application/json")
 
-        except Exception, e:
+        except Exception as e:
             return HttpResponse(dumps({'status': 'ko', 'error': str(e)}), content_type="application/json")
 
     return HttpResponseBadRequest(dumps({'status': 'ko'}), content_type="application/json")
diff --git a/fir_api/urls.py b/fir_api/urls.py
@@ -4,6 +4,8 @@
 
 from fir_api import views
 
+app_name='fir_api'
+
 # automatic URL routing for API
 # include login URLs for the browsable API.
 router = routers.DefaultRouter(trailing_slash=False)

diff --git a/fir_api/views.py b/fir_api/views.py
@@ -1,5 +1,5 @@
 # for token Generation
-import StringIO
+import io
 
 from django.conf import settings
 from django.db.models.signals import post_save
@@ -15,7 +15,7 @@
 from rest_framework.authtoken.models import Token
 from rest_framework.mixins import ListModelMixin, RetrieveModelMixin
 from rest_framework import viewsets
-from rest_framework.decorators import detail_route
+from rest_framework.decorators import action
 from rest_framework import renderers
 
 from fir_api.serializers import UserSerializer, IncidentSerializer, ArtifactSerializer, FileSerializer
@@ -83,17 +83,17 @@ class FileViewSet(ListModelMixin, RetrieveModelMixin, viewsets.GenericViewSet):
     serializer_class = FileSerializer
     permission_classes = (IsAuthenticated, IsIncidentHandler)
 
-    @detail_route(renderer_classes=[renderers.StaticHTMLRenderer])
+    @action(detail=True, renderer_classes=[renderers.StaticHTMLRenderer])
     def download(self, request, pk):
         return do_download(request, pk)
 
-    @detail_route(methods=["POST"])
+    @action(detail=True, methods=["POST"])
     def upload(self, request, pk):
         files = request.data['files']
         incident = get_object_or_404(Incident, pk=pk)
         files_added = []
         for i, file in enumerate(files):
-            file_obj = FileWrapper(StringIO.StringIO(file['content']))
+            file_obj = FileWrapper(io.StringIO(file['content']))
             file_obj.name = file['filename']
             description = file['description']
             f = handle_uploaded_file(file_obj, description, incident)

diff --git a/fir_artifacts/migrations/0002_create_artifacts.py b/fir_artifacts/migrations/0002_create_artifacts.py
@@ -44,7 +44,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='file',
             name='content_type',
-            field=models.ForeignKey(to='contenttypes.ContentType', null=True),
+            field=models.ForeignKey(to='contenttypes.ContentType', on_delete=models.CASCADE, null=True),
             preserve_default=True,
         ),
         migrations.AddField(

diff --git a/fir_artifacts/models.py b/fir_artifacts/models.py
@@ -8,15 +8,15 @@ class ArtifactBlacklistItem(models.Model):
     type = models.CharField(max_length=20)
     value = models.CharField(max_length=200)
 
-    def __unicode__(self):
+    def __str__(self):
         return self.value
 
 
 class Artifact(ManyLinkableModel):
     type = models.CharField(max_length=20)
     value = models.TextField()
 
-    def __unicode__(self):
+    def __str__(self):
         display = self.value
         if self.relations.count() > 1:
             display += " (%s)" % self.relations.count()
@@ -34,7 +34,7 @@ class File(OneLinkableModel):
     file = models.FileField(upload_to=upload_path)
     date = models.DateTimeField(auto_now_add=True)
 
-    def __unicode__(self):
+    def __str__(self):
         return unicode(self.file.name)
 
     def getfilename(self):

diff --git a/fir_artifacts/templates/fir_artifacts/plugins/details_static.html b/fir_artifacts/templates/fir_artifacts/plugins/details_static.html
@@ -1,2 +1,2 @@
-{% load staticfiles %}
+{% load static %}
 <script src="{% static "fir_artifacts/js/tooltips.js" %}"></script>
diff --git a/fir_artifacts/urls.py b/fir_artifacts/urls.py
@@ -2,6 +2,8 @@
 
 from fir_artifacts import views
 
+app_name='fir_artifacts'
+
 urlpatterns = [
     url(r'^(?P<artifact_id>\d+)/detach/(?P<relation_name>\w+)/(?P<relation_id>\d+)/$', views.detach_artifact, name='detach'),
     url(r'^(?P<artifact_id>\d+)/correlations/$', views.artifacts_correlations, name='correlations'),

diff --git a/fir_artifacts_enrichment/models.py b/fir_artifacts_enrichment/models.py
@@ -11,7 +11,7 @@ class ArtifactEnrichment(models.Model):
     name = models.CharField(max_length=100)
     raw = models.TextField()
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
 

diff --git a/fir_artifacts_enrichment/requirements.txt b/fir_artifacts_enrichment/requirements.txt
@@ -1 +1 @@
-abuse_finder
+git+https://github.com/certsocietegenerale/abuse_finder.git#egg=abuse_finder
diff --git a/fir_artifacts_enrichment/tasks.py b/fir_artifacts_enrichment/tasks.py
@@ -15,7 +15,7 @@
 @celery_app.task
 def enrich_artifact(artifact_id):
     artifact = Artifact.objects.get(pk=artifact_id)
-    print "Enrichment for {}".format(artifact.value)
+    print("Enrichment for {}".format(artifact.value))
 
     if artifact.type in ENRICHMENT_FUNCTIONS:
         results = ENRICHMENT_FUNCTIONS[artifact.type](artifact.value)

diff --git a/fir_notifications/models.py b/fir_notifications/models.py
@@ -3,17 +3,14 @@
 from django.db import models
 from django.conf import settings
 from django.db.models.signals import post_save
-from django.utils.encoding import python_2_unicode_compatible
 from django.utils.translation import ugettext_lazy as _
 from fir_notifications.decorators import notification_event
 
 from fir_notifications.registry import registry
 from incidents.models import model_created, Incident, model_updated, Comments, model_status_changed
 
-
-@python_2_unicode_compatible
 class MethodConfiguration(models.Model):
-    user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name='method_preferences', verbose_name=_('user'))
+    user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name='method_preferences', verbose_name=_('user'))
     key = models.CharField(max_length=60, choices=registry.get_method_choices(), verbose_name=_('method'))
     value = models.TextField(verbose_name=_('configuration'))
 
@@ -39,10 +36,8 @@ class Meta:
         verbose_name = _('notification template')
         verbose_name_plural = _('notification templates')
 
-
-@python_2_unicode_compatible
 class NotificationPreference(models.Model):
-    user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name='notification_preferences', verbose_name=_('user'))
+    user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name='notification_preferences', verbose_name=_('user'))
     event = models.CharField(max_length=60, verbose_name=_('event'))
     method = models.CharField(max_length=60, verbose_name=_('method'))
     business_lines = models.ManyToManyField('incidents.BusinessLine', related_name='+', blank=True,

diff --git a/fir_notifications/templates/fir_notifications/subscriptions.html b/fir_notifications/templates/fir_notifications/subscriptions.html
@@ -1,6 +1,6 @@
 {% load i18n %}
 {% load notifications %}
-{% load staticfiles %}
+{% load static %}
 
 <div class="col-sm-8 col-sm-offset-2">
     <div class="panel panel-default">

diff --git a/fir_notifications/urls.py b/fir_notifications/urls.py
@@ -2,6 +2,7 @@
 
 from fir_notifications import views
 
+app_name='fir_notifications'
 
 urlpatterns = [
     url(r'^subscriptions$', views.subscriptions, name='subscriptions'),

diff --git a/fir_nuggets/migrations/0001_initial.py b/fir_nuggets/migrations/0001_initial.py
@@ -24,8 +24,8 @@ class Migration(migrations.Migration):
                 ('start_timestamp', models.DateTimeField(default=datetime.datetime.now, null=True, blank=True)),
                 ('end_timestamp', models.DateTimeField(null=True, blank=True)),
                 ('interpretation', models.TextField()),
-                ('found_by', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
-                ('incident', models.ForeignKey(to='incidents.Incident')),
+                ('found_by', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.deletion.CASCADE)),
+                ('incident', models.ForeignKey(to='incidents.Incident', on_delete=models.deletion.CASCADE)),
             ],
             options={
             },
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		abuse_finder
		git+https://github.com/certsocietegenerale/abuse_finder.git#egg=abuse_finder